2026-02-22T22:51:11.576 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-22T22:51:11.596 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T22:51:11.678 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284 branch: wip-yuri6-testing-2026-02-22-1629-squid description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-bitmap supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-exclusive-lock} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '64284' kernel: branch: distro kdb: true sha1: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode name: yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-yuri6-testing-2026-02-22-1629-squid ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.4: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.5: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.6: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 selinux: allowlist: - scontext=system_u:system_r:getty_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-yuri6-testing-2026-02-22-1629-squid sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 9196 sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-yuri6-testing-2026-02-22-1629-squid suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 targets: trial060.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYMkwFUQ6cXE+ZjCzk3UVcU7g01xRvGge6562s1gf58Hws4h6Io7AwM+0Sbn9MFfRklivaxmFE6mipfm0hckrE= trial082.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPglXcHDxID8MMv+LLgiKjwWZtASzV0uUItaQbBbwe7w6JE63GHaCY+64iPiIcJLBXEE5mo+m1+LkfyxusecEPE= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror.sh env: CEPH_ARGS: '' RBD_IMAGE_FEATURES: layering,exclusive-lock RBD_MIRROR_CONFIG_KEY: '1' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-22_19:35:38 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653 2026-02-22T22:51:11.678 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa; will attempt to use it 2026-02-22T22:51:11.679 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa/tasks 2026-02-22T22:51:11.679 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-22T22:51:11.680 INFO:teuthology.task.internal:Checking packages... 2026-02-22T22:51:11.680 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'f90f4eaebb28b619cf76e80cbbe997656da69579' 2026-02-22T22:51:11.680 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-22T22:51:11.680 INFO:teuthology.packaging:ref: None 2026-02-22T22:51:11.680 INFO:teuthology.packaging:tag: None 2026-02-22T22:51:11.680 INFO:teuthology.packaging:branch: wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:51:11.680 INFO:teuthology.packaging:sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:51:11.681 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:51:11.800 INFO:teuthology.task.internal:Found packages for ceph version 19.2.3-689.gf90f4eae 2026-02-22T22:51:11.802 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-22T22:51:11.819 INFO:teuthology.task.internal:no buildpackages task found 2026-02-22T22:51:11.820 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-22T22:51:11.833 INFO:teuthology.task.internal:Saving configuration 2026-02-22T22:51:11.848 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-22T22:51:11.860 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-22T22:51:11.937 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial060.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 22:41:04.996441', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYMkwFUQ6cXE+ZjCzk3UVcU7g01xRvGge6562s1gf58Hws4h6Io7AwM+0Sbn9MFfRklivaxmFE6mipfm0hckrE='} 2026-02-22T22:51:11.994 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial082.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 22:41:04.995754', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPglXcHDxID8MMv+LLgiKjwWZtASzV0uUItaQbBbwe7w6JE63GHaCY+64iPiIcJLBXEE5mo+m1+LkfyxusecEPE='} 2026-02-22T22:51:11.994 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-22T22:51:12.005 INFO:teuthology.task.internal:roles: ubuntu@trial060.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-22T22:51:12.005 INFO:teuthology.task.internal:roles: ubuntu@trial082.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-22T22:51:12.005 INFO:teuthology.run_tasks:Running task console_log... 2026-02-22T22:51:12.141 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x70e9ce2e6d40>, signals=[15]) 2026-02-22T22:51:12.141 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-22T22:51:12.152 INFO:teuthology.task.internal:Opening connections... 2026-02-22T22:51:12.152 DEBUG:teuthology.task.internal:connecting to ubuntu@trial060.front.sepia.ceph.com 2026-02-22T22:51:12.152 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial060.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:51:12.218 DEBUG:teuthology.task.internal:connecting to ubuntu@trial082.front.sepia.ceph.com 2026-02-22T22:51:12.218 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:51:12.283 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-22T22:51:12.298 DEBUG:teuthology.orchestra.run.trial060:> uname -m 2026-02-22T22:51:12.315 INFO:teuthology.orchestra.run.trial060.stdout:x86_64 2026-02-22T22:51:12.316 DEBUG:teuthology.orchestra.run.trial060:> cat /etc/os-release 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:NAME="CentOS Stream" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:VERSION="9" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:ID="centos" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:ID_LIKE="rhel fedora" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:VERSION_ID="9" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:PLATFORM_ID="platform:el9" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:ANSI_COLOR="0;31" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:LOGO="fedora-logo-icon" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:HOME_URL="https://centos.org/" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-22T22:51:12.371 INFO:teuthology.orchestra.run.trial060.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-22T22:51:12.372 INFO:teuthology.lock.ops:Updating trial060.front.sepia.ceph.com on lock server 2026-02-22T22:51:12.450 DEBUG:teuthology.orchestra.run.trial082:> uname -m 2026-02-22T22:51:12.467 INFO:teuthology.orchestra.run.trial082.stdout:x86_64 2026-02-22T22:51:12.467 DEBUG:teuthology.orchestra.run.trial082:> cat /etc/os-release 2026-02-22T22:51:12.522 INFO:teuthology.orchestra.run.trial082.stdout:NAME="CentOS Stream" 2026-02-22T22:51:12.522 INFO:teuthology.orchestra.run.trial082.stdout:VERSION="9" 2026-02-22T22:51:12.522 INFO:teuthology.orchestra.run.trial082.stdout:ID="centos" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:ID_LIKE="rhel fedora" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:VERSION_ID="9" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:PLATFORM_ID="platform:el9" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:ANSI_COLOR="0;31" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:LOGO="fedora-logo-icon" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:HOME_URL="https://centos.org/" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-22T22:51:12.523 INFO:teuthology.orchestra.run.trial082.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-22T22:51:12.523 INFO:teuthology.lock.ops:Updating trial082.front.sepia.ceph.com on lock server 2026-02-22T22:51:12.597 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-22T22:51:12.610 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-22T22:51:12.620 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-22T22:51:12.620 DEBUG:teuthology.orchestra.run.trial060:> test '!' -e /home/ubuntu/cephtest 2026-02-22T22:51:12.622 DEBUG:teuthology.orchestra.run.trial082:> test '!' -e /home/ubuntu/cephtest 2026-02-22T22:51:12.638 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-22T22:51:12.652 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-22T22:51:12.653 DEBUG:teuthology.orchestra.run.trial060:> test -z $(ls -A /var/lib/ceph) 2026-02-22T22:51:12.678 DEBUG:teuthology.orchestra.run.trial082:> test -z $(ls -A /var/lib/ceph) 2026-02-22T22:51:12.695 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-22T22:51:12.851 INFO:teuthology.run_tasks:Running task kernel... 2026-02-22T22:51:12.871 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-22T22:51:12.871 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-22T22:51:12.871 DEBUG:teuthology.orchestra.run.trial060:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T22:51:12.872 DEBUG:teuthology.orchestra.run.trial082:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T22:51:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:51:12.888 DEBUG:teuthology.orchestra.run.trial060:> uname -r 2026-02-22T22:51:12.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:51:12.888 DEBUG:teuthology.orchestra.run.trial082:> uname -r 2026-02-22T22:51:12.943 INFO:teuthology.orchestra.run.trial060.stdout:5.14.0-681.el9.x86_64 2026-02-22T22:51:12.943 INFO:teuthology.task.kernel:Running kernel on trial060: 5.14.0-681.el9.x86_64 2026-02-22T22:51:12.943 DEBUG:teuthology.orchestra.run.trial060:> sudo yum install -y kernel 2026-02-22T22:51:12.944 INFO:teuthology.orchestra.run.trial082.stdout:5.14.0-681.el9.x86_64 2026-02-22T22:51:12.944 INFO:teuthology.task.kernel:Running kernel on trial082: 5.14.0-681.el9.x86_64 2026-02-22T22:51:12.944 DEBUG:teuthology.orchestra.run.trial082:> sudo yum install -y kernel 2026-02-22T22:51:14.015 INFO:teuthology.orchestra.run.trial082.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.9 MB 00:00 2026-02-22T22:51:14.361 INFO:teuthology.orchestra.run.trial060.stdout:CentOS Stream 9 - BaseOS 8.7 MB/s | 8.9 MB 00:01 2026-02-22T22:51:14.878 INFO:teuthology.orchestra.run.trial082.stdout:CentOS Stream 9 - AppStream 80 MB/s | 27 MB 00:00 2026-02-22T22:51:17.523 INFO:teuthology.orchestra.run.trial082.stdout:CentOS Stream 9 - CRB 28 MB/s | 7.8 MB 00:00 2026-02-22T22:51:18.361 INFO:teuthology.orchestra.run.trial082.stdout:CentOS Stream 9 - Extras packages 195 kB/s | 20 kB 00:00 2026-02-22T22:51:18.492 INFO:teuthology.orchestra.run.trial060.stdout:CentOS Stream 9 - AppStream 7.4 MB/s | 27 MB 00:03 2026-02-22T22:51:19.108 INFO:teuthology.orchestra.run.trial082.stdout:Extra Packages for Enterprise Linux 30 MB/s | 20 MB 00:00 2026-02-22T22:51:22.261 INFO:teuthology.orchestra.run.trial082.stdout:lab-extras 1.8 MB/s | 50 kB 00:00 2026-02-22T22:51:23.272 INFO:teuthology.orchestra.run.trial082.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-22T22:51:23.272 INFO:teuthology.orchestra.run.trial082.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-22T22:51:23.291 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:51:23.292 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:51:23.292 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:51:23.317 DEBUG:teuthology.orchestra.run.trial082:> echo no | sudo yum reinstall kernel || true 2026-02-22T22:51:23.662 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:01 ago on Sun 22 Feb 2026 10:51:22 PM UTC. 2026-02-22T22:51:23.743 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:51:23.744 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:51:23.745 INFO:teuthology.orchestra.run.trial082.stdout: Package Architecture Version Repository Size 2026-02-22T22:51:23.745 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:51:23.745 INFO:teuthology.orchestra.run.trial082.stdout:Reinstalling: 2026-02-22T22:51:23.745 INFO:teuthology.orchestra.run.trial082.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:51:23.745 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:51:23.745 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:51:23.745 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:51:23.745 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:51:23.745 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 803 k 2026-02-22T22:51:23.746 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 0 2026-02-22T22:51:23.746 INFO:teuthology.orchestra.run.trial082.stderr:Operation aborted. 2026-02-22T22:51:23.764 INFO:teuthology.orchestra.run.trial082.stdout:Is this ok [y/N]: 2026-02-22T22:51:23.765 DEBUG:teuthology.orchestra.run.trial082:> sudo yum reinstall -y kernel || true 2026-02-22T22:51:24.066 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 10:51:22 PM UTC. 2026-02-22T22:51:24.146 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:51:24.146 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout: Package Architecture Version Repository Size 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout:Reinstalling: 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 803 k 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 0 2026-02-22T22:51:24.147 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:51:24.606 INFO:teuthology.orchestra.run.trial082.stdout:kernel-5.14.0-681.el9.x86_64.rpm 3.9 MB/s | 803 kB 00:00 2026-02-22T22:51:24.607 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:51:24.607 INFO:teuthology.orchestra.run.trial082.stdout:Total 1.7 MB/s | 803 kB 00:00 2026-02-22T22:51:24.616 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:51:24.649 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:51:24.650 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:51:24.654 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:51:24.654 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:51:24.678 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:51:24.682 INFO:teuthology.orchestra.run.trial082.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:51:24.682 INFO:teuthology.orchestra.run.trial082.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:51:24.789 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:51:24.789 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:51:24.841 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:51:24.841 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:51:24.841 INFO:teuthology.orchestra.run.trial082.stdout:Reinstalled: 2026-02-22T22:51:24.841 INFO:teuthology.orchestra.run.trial082.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-22T22:51:24.842 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:51:24.842 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:51:24.868 DEBUG:teuthology.orchestra.run.trial082:> rpm -q kernel | sort -rV | head -n 1 2026-02-22T22:51:24.934 INFO:teuthology.orchestra.run.trial082.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-22T22:51:24.935 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-22T22:51:24.935 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T22:51:24.936 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-22T22:51:24.936 DEBUG:teuthology.orchestra.run.trial082:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T22:51:24.999 INFO:teuthology.orchestra.run.trial082.stdout:ttyS1 2026-02-22T22:51:25.029 DEBUG:teuthology.parallel:result is None 2026-02-22T22:51:29.416 INFO:teuthology.orchestra.run.trial060.stdout:CentOS Stream 9 - CRB 930 kB/s | 7.8 MB 00:08 2026-02-22T22:51:31.033 INFO:teuthology.orchestra.run.trial060.stdout:CentOS Stream 9 - Extras packages 23 kB/s | 20 kB 00:00 2026-02-22T22:51:31.595 INFO:teuthology.orchestra.run.trial060.stdout:Extra Packages for Enterprise Linux 41 MB/s | 20 MB 00:00 2026-02-22T22:51:34.764 INFO:teuthology.orchestra.run.trial060.stdout:lab-extras 1.5 MB/s | 50 kB 00:00 2026-02-22T22:51:35.781 INFO:teuthology.orchestra.run.trial060.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-22T22:51:35.782 INFO:teuthology.orchestra.run.trial060.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-22T22:51:35.800 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:51:35.801 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:51:35.801 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:51:35.824 DEBUG:teuthology.orchestra.run.trial060:> echo no | sudo yum reinstall kernel || true 2026-02-22T22:51:36.123 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 10:51:34 PM UTC. 2026-02-22T22:51:36.203 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:51:36.204 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:51:36.204 INFO:teuthology.orchestra.run.trial060.stdout: Package Architecture Version Repository Size 2026-02-22T22:51:36.204 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:51:36.205 INFO:teuthology.orchestra.run.trial060.stdout:Reinstalling: 2026-02-22T22:51:36.205 INFO:teuthology.orchestra.run.trial060.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:51:36.205 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:51:36.205 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:51:36.205 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:51:36.205 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:51:36.205 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 803 k 2026-02-22T22:51:36.205 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 0 2026-02-22T22:51:36.205 INFO:teuthology.orchestra.run.trial060.stderr:Operation aborted. 2026-02-22T22:51:36.224 INFO:teuthology.orchestra.run.trial060.stdout:Is this ok [y/N]: 2026-02-22T22:51:36.224 DEBUG:teuthology.orchestra.run.trial060:> sudo yum reinstall -y kernel || true 2026-02-22T22:51:36.520 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 10:51:34 PM UTC. 2026-02-22T22:51:36.599 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:51:36.600 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:51:36.601 INFO:teuthology.orchestra.run.trial060.stdout: Package Architecture Version Repository Size 2026-02-22T22:51:36.601 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:51:36.601 INFO:teuthology.orchestra.run.trial060.stdout:Reinstalling: 2026-02-22T22:51:36.601 INFO:teuthology.orchestra.run.trial060.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:51:36.601 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:51:36.601 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:51:36.601 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:51:36.601 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:51:36.601 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 803 k 2026-02-22T22:51:36.601 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 0 2026-02-22T22:51:36.602 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:51:37.531 INFO:teuthology.orchestra.run.trial060.stdout:kernel-5.14.0-681.el9.x86_64.rpm 1.2 MB/s | 803 kB 00:00 2026-02-22T22:51:37.531 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:51:37.531 INFO:teuthology.orchestra.run.trial060.stdout:Total 863 kB/s | 803 kB 00:00 2026-02-22T22:51:37.541 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:51:37.577 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:51:37.577 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:51:37.582 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:51:37.582 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:51:37.606 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:51:37.610 INFO:teuthology.orchestra.run.trial060.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:51:37.610 INFO:teuthology.orchestra.run.trial060.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:51:37.718 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:51:37.718 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:51:37.773 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:51:37.774 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:51:37.774 INFO:teuthology.orchestra.run.trial060.stdout:Reinstalled: 2026-02-22T22:51:37.774 INFO:teuthology.orchestra.run.trial060.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-22T22:51:37.774 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:51:37.774 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:51:37.802 DEBUG:teuthology.orchestra.run.trial060:> rpm -q kernel | sort -rV | head -n 1 2026-02-22T22:51:37.826 INFO:teuthology.orchestra.run.trial060.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-22T22:51:37.827 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-22T22:51:37.827 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T22:51:37.827 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-22T22:51:37.827 DEBUG:teuthology.orchestra.run.trial060:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T22:51:37.891 INFO:teuthology.orchestra.run.trial060.stdout:ttyS1 2026-02-22T22:51:37.916 DEBUG:teuthology.parallel:result is None 2026-02-22T22:51:37.916 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-22T22:51:37.929 INFO:teuthology.task.internal:Creating test directory... 2026-02-22T22:51:37.930 DEBUG:teuthology.orchestra.run.trial060:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T22:51:37.959 DEBUG:teuthology.orchestra.run.trial082:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T22:51:37.977 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-22T22:51:37.988 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-22T22:51:38.002 INFO:teuthology.task.internal:Creating archive directory... 2026-02-22T22:51:38.002 DEBUG:teuthology.orchestra.run.trial060:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T22:51:38.015 DEBUG:teuthology.orchestra.run.trial082:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T22:51:38.044 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-22T22:51:38.055 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-22T22:51:38.056 DEBUG:teuthology.orchestra.run.trial060:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T22:51:38.072 DEBUG:teuthology.orchestra.run.trial082:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T22:51:38.094 INFO:teuthology.orchestra.run.trial060.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:51:38.100 INFO:teuthology.orchestra.run.trial082.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:51:38.103 INFO:teuthology.orchestra.run.trial060.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:51:38.108 INFO:teuthology.orchestra.run.trial082.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:51:38.110 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-22T22:51:38.122 INFO:teuthology.task.internal:Configuring sudo... 2026-02-22T22:51:38.122 DEBUG:teuthology.orchestra.run.trial060:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T22:51:38.148 DEBUG:teuthology.orchestra.run.trial082:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T22:51:38.175 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-22T22:51:38.192 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-22T22:51:38.192 DEBUG:teuthology.orchestra.run.trial060:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T22:51:38.213 DEBUG:teuthology.orchestra.run.trial082:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T22:51:38.232 DEBUG:teuthology.orchestra.run.trial060:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:51:38.387 DEBUG:teuthology.orchestra.run.trial060:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:51:38.413 DEBUG:teuthology.orchestra.run.trial060:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:51:38.558 DEBUG:teuthology.orchestra.run.trial060:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:51:38.582 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:51:38.582 DEBUG:teuthology.orchestra.run.trial060:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T22:51:38.647 DEBUG:teuthology.orchestra.run.trial082:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:51:38.770 DEBUG:teuthology.orchestra.run.trial082:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:51:38.797 DEBUG:teuthology.orchestra.run.trial082:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:51:38.938 DEBUG:teuthology.orchestra.run.trial082:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:51:38.961 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:51:38.961 DEBUG:teuthology.orchestra.run.trial082:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T22:51:39.025 DEBUG:teuthology.orchestra.run.trial060:> sudo service rsyslog restart 2026-02-22T22:51:39.027 DEBUG:teuthology.orchestra.run.trial082:> sudo service rsyslog restart 2026-02-22T22:51:39.055 INFO:teuthology.orchestra.run.trial060.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:51:39.092 INFO:teuthology.orchestra.run.trial082.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:51:39.526 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-22T22:51:39.539 INFO:teuthology.task.internal:Starting timer... 2026-02-22T22:51:39.539 INFO:teuthology.run_tasks:Running task pcp... 2026-02-22T22:51:39.557 INFO:teuthology.run_tasks:Running task selinux... 2026-02-22T22:51:39.570 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:getty_t:s0']} 2026-02-22T22:51:39.570 DEBUG:teuthology.orchestra.run.trial060:> sudo service auditd rotate 2026-02-22T22:51:39.600 INFO:teuthology.orchestra.run.trial060.stdout:Rotating logs: 2026-02-22T22:51:39.602 DEBUG:teuthology.orchestra.run.trial082:> sudo service auditd rotate 2026-02-22T22:51:39.632 INFO:teuthology.orchestra.run.trial082.stdout:Rotating logs: 2026-02-22T22:51:39.635 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-22T22:51:39.635 DEBUG:teuthology.orchestra.run.trial060:> /usr/sbin/getenforce 2026-02-22T22:51:39.660 INFO:teuthology.orchestra.run.trial060.stdout:Permissive 2026-02-22T22:51:39.660 DEBUG:teuthology.orchestra.run.trial082:> /usr/sbin/getenforce 2026-02-22T22:51:39.693 INFO:teuthology.orchestra.run.trial082.stdout:Permissive 2026-02-22T22:51:39.694 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@trial060.front.sepia.ceph.com': 'permissive', 'ubuntu@trial082.front.sepia.ceph.com': 'permissive'} 2026-02-22T22:51:39.694 DEBUG:teuthology.orchestra.run.trial060:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:51:39.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:51:39.725 DEBUG:teuthology.orchestra.run.trial082:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:51:39.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:51:39.761 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-22T22:51:39.762 DEBUG:teuthology.orchestra.run.trial060:> sudo /usr/sbin/setenforce permissive 2026-02-22T22:51:39.790 DEBUG:teuthology.orchestra.run.trial082:> sudo /usr/sbin/setenforce permissive 2026-02-22T22:51:39.824 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-22T22:51:39.837 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2026-02-22T22:51:39.839 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2026-02-22T22:51:39.950 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-22T22:51:39.958 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-22T22:51:39.959 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial060.front.sepia.ceph.com,trial082.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-22T22:54:26.639 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial060.front.sepia.ceph.com'), Remote(name='ubuntu@trial082.front.sepia.ceph.com')] 2026-02-22T22:54:26.640 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial060.front.sepia.ceph.com' 2026-02-22T22:54:26.641 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial060.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:54:26.753 DEBUG:teuthology.orchestra.run.trial060:> true 2026-02-22T22:54:26.831 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial060.front.sepia.ceph.com' 2026-02-22T22:54:26.831 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial082.front.sepia.ceph.com' 2026-02-22T22:54:26.832 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial082.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:54:26.910 DEBUG:teuthology.orchestra.run.trial082:> true 2026-02-22T22:54:26.986 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial082.front.sepia.ceph.com' 2026-02-22T22:54:26.986 INFO:teuthology.run_tasks:Running task clock... 2026-02-22T22:54:26.999 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-22T22:54:26.999 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T22:54:27.000 DEBUG:teuthology.orchestra.run.trial060:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:54:27.002 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T22:54:27.002 DEBUG:teuthology.orchestra.run.trial082:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:54:27.027 INFO:teuthology.orchestra.run.trial060.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-22T22:54:27.039 INFO:teuthology.orchestra.run.trial060.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-22T22:54:27.053 INFO:teuthology.orchestra.run.trial082.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-22T22:54:27.060 INFO:teuthology.orchestra.run.trial060.stderr:sudo: ntpd: command not found 2026-02-22T22:54:27.065 INFO:teuthology.orchestra.run.trial082.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-22T22:54:27.070 INFO:teuthology.orchestra.run.trial060.stdout:506 Cannot talk to daemon 2026-02-22T22:54:27.081 INFO:teuthology.orchestra.run.trial060.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-22T22:54:27.086 INFO:teuthology.orchestra.run.trial082.stderr:sudo: ntpd: command not found 2026-02-22T22:54:27.094 INFO:teuthology.orchestra.run.trial060.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-22T22:54:27.096 INFO:teuthology.orchestra.run.trial082.stdout:506 Cannot talk to daemon 2026-02-22T22:54:27.108 INFO:teuthology.orchestra.run.trial082.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-22T22:54:27.120 INFO:teuthology.orchestra.run.trial082.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-22T22:54:27.136 INFO:teuthology.orchestra.run.trial060.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:54:27.155 INFO:teuthology.orchestra.run.trial060.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:54:27.155 INFO:teuthology.orchestra.run.trial060.stdout:=============================================================================== 2026-02-22T22:54:27.155 INFO:teuthology.orchestra.run.trial060.stdout:^? 172-234-44-141.ip.linode> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:54:27.155 INFO:teuthology.orchestra.run.trial060.stdout:^? time-ord.far.net 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:54:27.155 INFO:teuthology.orchestra.run.trial060.stdout:^? ntp01-baq.provider0.inf 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:54:27.155 INFO:teuthology.orchestra.run.trial060.stdout:^? ntp1.glypnod.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:54:27.163 INFO:teuthology.orchestra.run.trial082.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:54:27.346 INFO:teuthology.orchestra.run.trial082.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:54:27.347 INFO:teuthology.orchestra.run.trial082.stdout:=============================================================================== 2026-02-22T22:54:27.347 INFO:teuthology.orchestra.run.trial082.stdout:^? ntp2.zeus.frumentum.media 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:54:27.347 INFO:teuthology.orchestra.run.trial082.stdout:^? 72-14-186-59.ip.linodeus> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:54:27.347 INFO:teuthology.orchestra.run.trial082.stdout:^? 173.249.203.227 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:54:27.347 INFO:teuthology.orchestra.run.trial082.stdout:^? nyc3.us.ntp.li 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:54:27.348 INFO:teuthology.run_tasks:Running task install... 2026-02-22T22:54:27.363 DEBUG:teuthology.task.install:project ceph 2026-02-22T22:54:27.363 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}} 2026-02-22T22:54:27.363 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'} 2026-02-22T22:54:27.363 INFO:teuthology.task.install:Using flavor: default 2026-02-22T22:54:27.372 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-22T22:54:27.372 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-22T22:54:27.372 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'tag': None, 'wait_for_package': False} 2026-02-22T22:54:27.373 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:54:27.377 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'tag': None, 'wait_for_package': False} 2026-02-22T22:54:27.377 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:54:27.564 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/ 2026-02-22T22:54:27.564 INFO:teuthology.task.install.rpm:Package version is 19.2.3-689.gf90f4eae 2026-02-22T22:54:27.574 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/ 2026-02-22T22:54:27.574 INFO:teuthology.task.install.rpm:Package version is 19.2.3-689.gf90f4eae 2026-02-22T22:54:27.708 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-22T22:54:27.708 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:54:27.708 DEBUG:teuthology.orchestra.run.trial060:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-22T22:54:27.719 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-22T22:54:27.719 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:54:27.719 DEBUG:teuthology.orchestra.run.trial082:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-22T22:54:27.733 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-22T22:54:27.734 DEBUG:teuthology.orchestra.run.trial060:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f90f4eaebb28b619cf76e80cbbe997656da69579/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-22T22:54:27.745 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-22T22:54:27.745 DEBUG:teuthology.orchestra.run.trial082:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f90f4eaebb28b619cf76e80cbbe997656da69579/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-22T22:54:27.798 DEBUG:teuthology.orchestra.run.trial060:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-22T22:54:27.809 DEBUG:teuthology.orchestra.run.trial082:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-22T22:54:27.870 DEBUG:teuthology.orchestra.run.trial060:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-22T22:54:27.882 DEBUG:teuthology.orchestra.run.trial082:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-22T22:54:27.933 INFO:teuthology.orchestra.run.trial060.stdout:check_obsoletes = 1 2026-02-22T22:54:27.934 DEBUG:teuthology.orchestra.run.trial060:> sudo yum clean all 2026-02-22T22:54:27.946 INFO:teuthology.orchestra.run.trial082.stdout:check_obsoletes = 1 2026-02-22T22:54:27.948 DEBUG:teuthology.orchestra.run.trial082:> sudo yum clean all 2026-02-22T22:54:28.091 INFO:teuthology.orchestra.run.trial060.stdout:41 files removed 2026-02-22T22:54:28.105 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-radosgw 2026-02-22T22:54:28.108 INFO:teuthology.orchestra.run.trial082.stdout:41 files removed 2026-02-22T22:54:28.123 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-radosgw 2026-02-22T22:54:28.386 INFO:teuthology.orchestra.run.trial060.stdout:ceph packages for x86_64 618 kB/s | 86 kB 00:00 2026-02-22T22:54:28.397 INFO:teuthology.orchestra.run.trial082.stdout:ceph packages for x86_64 662 kB/s | 86 kB 00:00 2026-02-22T22:54:28.522 INFO:teuthology.orchestra.run.trial060.stdout:ceph noarch packages 110 kB/s | 12 kB 00:00 2026-02-22T22:54:28.530 INFO:teuthology.orchestra.run.trial082.stdout:ceph noarch packages 111 kB/s | 12 kB 00:00 2026-02-22T22:54:28.648 INFO:teuthology.orchestra.run.trial060.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-22T22:54:28.659 INFO:teuthology.orchestra.run.trial082.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-22T22:54:28.911 INFO:teuthology.orchestra.run.trial060.stdout:CentOS Stream 9 - BaseOS 36 MB/s | 8.9 MB 00:00 2026-02-22T22:54:29.385 INFO:teuthology.orchestra.run.trial082.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.9 MB 00:00 2026-02-22T22:54:32.693 INFO:teuthology.orchestra.run.trial060.stdout:CentOS Stream 9 - AppStream 8.2 MB/s | 27 MB 00:03 2026-02-22T22:54:32.915 INFO:teuthology.orchestra.run.trial082.stdout:CentOS Stream 9 - AppStream 8.9 MB/s | 27 MB 00:03 2026-02-22T22:54:36.510 INFO:teuthology.orchestra.run.trial082.stdout:CentOS Stream 9 - CRB 6.5 MB/s | 7.8 MB 00:01 2026-02-22T22:54:36.760 INFO:teuthology.orchestra.run.trial060.stdout:CentOS Stream 9 - CRB 4.6 MB/s | 7.8 MB 00:01 2026-02-22T22:54:37.350 INFO:teuthology.orchestra.run.trial082.stdout:CentOS Stream 9 - Extras packages 177 kB/s | 20 kB 00:00 2026-02-22T22:54:37.594 INFO:teuthology.orchestra.run.trial060.stdout:CentOS Stream 9 - Extras packages 190 kB/s | 20 kB 00:00 2026-02-22T22:54:38.065 INFO:teuthology.orchestra.run.trial082.stdout:Extra Packages for Enterprise Linux 31 MB/s | 20 MB 00:00 2026-02-22T22:54:38.191 INFO:teuthology.orchestra.run.trial060.stdout:Extra Packages for Enterprise Linux 38 MB/s | 20 MB 00:00 2026-02-22T22:54:41.272 INFO:teuthology.orchestra.run.trial082.stdout:lab-extras 2.1 MB/s | 50 kB 00:00 2026-02-22T22:54:41.394 INFO:teuthology.orchestra.run.trial060.stdout:lab-extras 2.5 MB/s | 50 kB 00:00 2026-02-22T22:54:42.479 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:54:42.480 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:54:42.480 INFO:teuthology.orchestra.run.trial082.stdout: Package Arch Version Repository Size 2026-02-22T22:54:42.480 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:54:42.480 INFO:teuthology.orchestra.run.trial082.stdout:Installing: 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout: ceph-radosgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 11 M 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout:Upgrading: 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout: librados2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.4 M 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout: librbd1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.2 M 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout:Installing dependencies: 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout: ceph-base x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.5 M 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout: ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 22 M 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout: ceph-selinux x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 25 k 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-22T22:54:42.481 INFO:teuthology.orchestra.run.trial082.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: libcephfs2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 999 k 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: libradosstriper1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 503 k 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: librgw2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.4 M 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-22T22:54:42.482 INFO:teuthology.orchestra.run.trial082.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: python3-ceph-argparse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 45 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: python3-ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 142 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: python3-cephfs x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 165 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: python3-rados x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 322 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: python3-rbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 303 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: python3-rgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 100 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-22T22:54:42.483 INFO:teuthology.orchestra.run.trial082.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-22T22:54:42.484 INFO:teuthology.orchestra.run.trial082.stdout:Installing weak dependencies: 2026-02-22T22:54:42.484 INFO:teuthology.orchestra.run.trial082.stdout: nvme-cli x86_64 2.16-1.el9 baseos 1.2 M 2026-02-22T22:54:42.484 INFO:teuthology.orchestra.run.trial082.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-22T22:54:42.484 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:42.484 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:54:42.484 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:54:42.484 INFO:teuthology.orchestra.run.trial082.stdout:Install 33 Packages 2026-02-22T22:54:42.484 INFO:teuthology.orchestra.run.trial082.stdout:Upgrade 2 Packages 2026-02-22T22:54:42.484 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:42.485 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 63 M 2026-02-22T22:54:42.485 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:54:42.594 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:54:42.596 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:54:42.596 INFO:teuthology.orchestra.run.trial060.stdout: Package Arch Version Repository Size 2026-02-22T22:54:42.596 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:54:42.596 INFO:teuthology.orchestra.run.trial060.stdout:Installing: 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout: ceph-radosgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 11 M 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout:Upgrading: 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout: librados2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.4 M 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout: librbd1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.2 M 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout:Installing dependencies: 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout: ceph-base x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.5 M 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout: ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 22 M 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout: ceph-selinux x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 25 k 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-22T22:54:42.597 INFO:teuthology.orchestra.run.trial060.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: libcephfs2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 999 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: libradosstriper1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 503 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: librgw2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.4 M 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-22T22:54:42.598 INFO:teuthology.orchestra.run.trial060.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: python3-ceph-argparse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 45 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: python3-ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 142 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: python3-cephfs x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 165 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: python3-rados x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 322 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: python3-rbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 303 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: python3-rgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 100 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-22T22:54:42.599 INFO:teuthology.orchestra.run.trial060.stdout:Installing weak dependencies: 2026-02-22T22:54:42.600 INFO:teuthology.orchestra.run.trial060.stdout: nvme-cli x86_64 2.16-1.el9 baseos 1.2 M 2026-02-22T22:54:42.600 INFO:teuthology.orchestra.run.trial060.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-22T22:54:42.600 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:42.600 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:54:42.600 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:54:42.600 INFO:teuthology.orchestra.run.trial060.stdout:Install 33 Packages 2026-02-22T22:54:42.600 INFO:teuthology.orchestra.run.trial060.stdout:Upgrade 2 Packages 2026-02-22T22:54:42.600 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:42.601 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 63 M 2026-02-22T22:54:42.601 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:54:43.038 INFO:teuthology.orchestra.run.trial082.stdout:(1/35): ceph-base-19.2.3-689.gf90f4eae.el9.x86_ 18 MB/s | 5.5 MB 00:00 2026-02-22T22:54:43.076 INFO:teuthology.orchestra.run.trial082.stdout:(2/35): ceph-selinux-19.2.3-689.gf90f4eae.el9.x 648 kB/s | 25 kB 00:00 2026-02-22T22:54:43.136 INFO:teuthology.orchestra.run.trial082.stdout:(3/35): libcephfs2-19.2.3-689.gf90f4eae.el9.x86 16 MB/s | 999 kB 00:00 2026-02-22T22:54:43.188 INFO:teuthology.orchestra.run.trial082.stdout:(4/35): libradosstriper1-19.2.3-689.gf90f4eae.e 9.5 MB/s | 503 kB 00:00 2026-02-22T22:54:43.418 INFO:teuthology.orchestra.run.trial082.stdout:(5/35): ceph-radosgw-19.2.3-689.gf90f4eae.el9.x 16 MB/s | 11 MB 00:00 2026-02-22T22:54:43.455 INFO:teuthology.orchestra.run.trial082.stdout:(6/35): python3-ceph-argparse-19.2.3-689.gf90f4 1.2 MB/s | 45 kB 00:00 2026-02-22T22:54:43.464 INFO:teuthology.orchestra.run.trial082.stdout:(7/35): librgw2-19.2.3-689.gf90f4eae.el9.x86_64 20 MB/s | 5.4 MB 00:00 2026-02-22T22:54:43.479 INFO:teuthology.orchestra.run.trial082.stdout:(8/35): python3-ceph-common-19.2.3-689.gf90f4ea 5.6 MB/s | 142 kB 00:00 2026-02-22T22:54:43.492 INFO:teuthology.orchestra.run.trial082.stdout:(9/35): python3-cephfs-19.2.3-689.gf90f4eae.el9 5.8 MB/s | 165 kB 00:00 2026-02-22T22:54:43.512 INFO:teuthology.orchestra.run.trial082.stdout:(10/35): python3-rados-19.2.3-689.gf90f4eae.el9 9.7 MB/s | 322 kB 00:00 2026-02-22T22:54:43.522 INFO:teuthology.orchestra.run.trial082.stdout:(11/35): python3-rbd-19.2.3-689.gf90f4eae.el9.x 9.7 MB/s | 303 kB 00:00 2026-02-22T22:54:43.539 INFO:teuthology.orchestra.run.trial082.stdout:(12/35): python3-rgw-19.2.3-689.gf90f4eae.el9.x 3.6 MB/s | 100 kB 00:00 2026-02-22T22:54:43.639 INFO:teuthology.orchestra.run.trial082.stdout:(13/35): ceph-common-19.2.3-689.gf90f4eae.el9.x 24 MB/s | 22 MB 00:00 2026-02-22T22:54:43.716 INFO:teuthology.orchestra.run.trial060.stdout:(1/35): ceph-base-19.2.3-689.gf90f4eae.el9.x86_ 15 MB/s | 5.5 MB 00:00 2026-02-22T22:54:43.728 INFO:teuthology.orchestra.run.trial060.stdout:(2/35): ceph-selinux-19.2.3-689.gf90f4eae.el9.x 2.0 MB/s | 25 kB 00:00 2026-02-22T22:54:43.748 INFO:teuthology.orchestra.run.trial060.stdout:(3/35): libcephfs2-19.2.3-689.gf90f4eae.el9.x86 50 MB/s | 999 kB 00:00 2026-02-22T22:54:43.761 INFO:teuthology.orchestra.run.trial060.stdout:(4/35): libradosstriper1-19.2.3-689.gf90f4eae.e 36 MB/s | 503 kB 00:00 2026-02-22T22:54:43.873 INFO:teuthology.orchestra.run.trial060.stdout:(5/35): librgw2-19.2.3-689.gf90f4eae.el9.x86_64 48 MB/s | 5.4 MB 00:00 2026-02-22T22:54:43.886 INFO:teuthology.orchestra.run.trial060.stdout:(6/35): python3-ceph-argparse-19.2.3-689.gf90f4 3.7 MB/s | 45 kB 00:00 2026-02-22T22:54:43.897 INFO:teuthology.orchestra.run.trial060.stdout:(7/35): python3-ceph-common-19.2.3-689.gf90f4ea 12 MB/s | 142 kB 00:00 2026-02-22T22:54:43.910 INFO:teuthology.orchestra.run.trial060.stdout:(8/35): python3-cephfs-19.2.3-689.gf90f4eae.el9 13 MB/s | 165 kB 00:00 2026-02-22T22:54:43.922 INFO:teuthology.orchestra.run.trial060.stdout:(9/35): python3-rados-19.2.3-689.gf90f4eae.el9. 25 MB/s | 322 kB 00:00 2026-02-22T22:54:43.935 INFO:teuthology.orchestra.run.trial060.stdout:(10/35): python3-rbd-19.2.3-689.gf90f4eae.el9.x 24 MB/s | 303 kB 00:00 2026-02-22T22:54:43.946 INFO:teuthology.orchestra.run.trial060.stdout:(11/35): python3-rgw-19.2.3-689.gf90f4eae.el9.x 8.3 MB/s | 100 kB 00:00 2026-02-22T22:54:44.081 INFO:teuthology.orchestra.run.trial082.stdout:(14/35): mailcap-2.1.49-5.el9.noarch.rpm 59 kB/s | 33 kB 00:00 2026-02-22T22:54:44.278 INFO:teuthology.orchestra.run.trial082.stdout:(15/35): boost-program-options-1.75.0-13.el9.x8 527 kB/s | 104 kB 00:00 2026-02-22T22:54:44.285 INFO:teuthology.orchestra.run.trial060.stdout:(12/35): ceph-radosgw-19.2.3-689.gf90f4eae.el9. 12 MB/s | 11 MB 00:00 2026-02-22T22:54:44.353 INFO:teuthology.orchestra.run.trial082.stdout:(16/35): pciutils-3.7.0-7.el9.x86_64.rpm 130 kB/s | 93 kB 00:00 2026-02-22T22:54:44.509 INFO:teuthology.orchestra.run.trial060.stdout:(13/35): ceph-common-19.2.3-689.gf90f4eae.el9.x 19 MB/s | 22 MB 00:01 2026-02-22T22:54:44.510 INFO:teuthology.orchestra.run.trial060.stdout:(14/35): mailcap-2.1.49-5.el9.noarch.rpm 59 kB/s | 33 kB 00:00 2026-02-22T22:54:44.533 INFO:teuthology.orchestra.run.trial082.stdout:(17/35): libnbd-1.20.3-4.el9.x86_64.rpm 641 kB/s | 164 kB 00:00 2026-02-22T22:54:44.574 INFO:teuthology.orchestra.run.trial060.stdout:(15/35): boost-program-options-1.75.0-13.el9.x8 1.6 MB/s | 104 kB 00:00 2026-02-22T22:54:44.610 INFO:teuthology.orchestra.run.trial082.stdout:(18/35): libpmemobj-1.12.1-1.el9.x86_64.rpm 623 kB/s | 160 kB 00:00 2026-02-22T22:54:44.617 INFO:teuthology.orchestra.run.trial060.stdout:(16/35): libnbd-1.20.3-4.el9.x86_64.rpm 3.7 MB/s | 164 kB 00:00 2026-02-22T22:54:44.632 INFO:teuthology.orchestra.run.trial082.stdout:(19/35): librabbitmq-0.11.0-7.el9.x86_64.rpm 458 kB/s | 45 kB 00:00 2026-02-22T22:54:44.642 INFO:teuthology.orchestra.run.trial060.stdout:(17/35): libpmemobj-1.12.1-1.el9.x86_64.rpm 6.4 MB/s | 160 kB 00:00 2026-02-22T22:54:44.664 INFO:teuthology.orchestra.run.trial060.stdout:(18/35): librabbitmq-0.11.0-7.el9.x86_64.rpm 2.0 MB/s | 45 kB 00:00 2026-02-22T22:54:44.709 INFO:teuthology.orchestra.run.trial060.stdout:(19/35): librdkafka-1.6.1-102.el9.x86_64.rpm 14 MB/s | 662 kB 00:00 2026-02-22T22:54:44.733 INFO:teuthology.orchestra.run.trial060.stdout:(20/35): lttng-ust-2.12.0-6.el9.x86_64.rpm 12 MB/s | 292 kB 00:00 2026-02-22T22:54:44.745 INFO:teuthology.orchestra.run.trial060.stdout:(21/35): pciutils-3.7.0-7.el9.x86_64.rpm 394 kB/s | 93 kB 00:00 2026-02-22T22:54:44.756 INFO:teuthology.orchestra.run.trial060.stdout:(22/35): qatlib-25.08.0-2.el9.x86_64.rpm 10 MB/s | 240 kB 00:00 2026-02-22T22:54:44.778 INFO:teuthology.orchestra.run.trial060.stdout:(23/35): qatzip-libs-1.3.1-1.el9.x86_64.rpm 3.0 MB/s | 66 kB 00:00 2026-02-22T22:54:44.808 INFO:teuthology.orchestra.run.trial060.stdout:(24/35): qatlib-service-25.08.0-2.el9.x86_64.rp 590 kB/s | 37 kB 00:00 2026-02-22T22:54:44.863 INFO:teuthology.orchestra.run.trial060.stdout:(25/35): gperftools-libs-2.9.1-3.el9.x86_64.rpm 3.6 MB/s | 308 kB 00:00 2026-02-22T22:54:44.879 INFO:teuthology.orchestra.run.trial060.stdout:(26/35): libarrow-doc-9.0.0-15.el9.noarch.rpm 1.6 MB/s | 25 kB 00:00 2026-02-22T22:54:44.893 INFO:teuthology.orchestra.run.trial060.stdout:(27/35): liboath-2.6.12-1.el9.x86_64.rpm 3.2 MB/s | 49 kB 00:00 2026-02-22T22:54:44.900 INFO:teuthology.orchestra.run.trial060.stdout:(28/35): libarrow-9.0.0-15.el9.x86_64.rpm 48 MB/s | 4.4 MB 00:00 2026-02-22T22:54:44.908 INFO:teuthology.orchestra.run.trial060.stdout:(29/35): libunwind-1.6.2-1.el9.x86_64.rpm 4.4 MB/s | 67 kB 00:00 2026-02-22T22:54:44.919 INFO:teuthology.orchestra.run.trial060.stdout:(30/35): parquet-libs-9.0.0-15.el9.x86_64.rpm 45 MB/s | 838 kB 00:00 2026-02-22T22:54:44.924 INFO:teuthology.orchestra.run.trial060.stdout:(31/35): re2-20211101-20.el9.x86_64.rpm 12 MB/s | 191 kB 00:00 2026-02-22T22:54:44.948 INFO:teuthology.orchestra.run.trial060.stdout:(32/35): thrift-0.15.0-4.el9.x86_64.rpm 54 MB/s | 1.6 MB 00:00 2026-02-22T22:54:45.036 INFO:teuthology.orchestra.run.trial060.stdout:(33/35): librados2-19.2.3-689.gf90f4eae.el9.x86 31 MB/s | 3.4 MB 00:00 2026-02-22T22:54:45.045 INFO:teuthology.orchestra.run.trial082.stdout:(20/35): lttng-ust-2.12.0-6.el9.x86_64.rpm 709 kB/s | 292 kB 00:00 2026-02-22T22:54:45.061 INFO:teuthology.orchestra.run.trial060.stdout:(34/35): librbd1-19.2.3-689.gf90f4eae.el9.x86_6 28 MB/s | 3.2 MB 00:00 2026-02-22T22:54:45.379 INFO:teuthology.orchestra.run.trial082.stdout:(21/35): qatlib-25.08.0-2.el9.x86_64.rpm 717 kB/s | 240 kB 00:00 2026-02-22T22:54:45.417 INFO:teuthology.orchestra.run.trial082.stdout:(22/35): librdkafka-1.6.1-102.el9.x86_64.rpm 821 kB/s | 662 kB 00:00 2026-02-22T22:54:45.482 INFO:teuthology.orchestra.run.trial082.stdout:(23/35): qatlib-service-25.08.0-2.el9.x86_64.rp 361 kB/s | 37 kB 00:00 2026-02-22T22:54:45.517 INFO:teuthology.orchestra.run.trial082.stdout:(24/35): qatzip-libs-1.3.1-1.el9.x86_64.rpm 664 kB/s | 66 kB 00:00 2026-02-22T22:54:45.540 INFO:teuthology.orchestra.run.trial082.stdout:(25/35): nvme-cli-2.16-1.el9.x86_64.rpm 589 kB/s | 1.2 MB 00:02 2026-02-22T22:54:45.654 INFO:teuthology.orchestra.run.trial082.stdout:(26/35): libarrow-doc-9.0.0-15.el9.noarch.rpm 218 kB/s | 25 kB 00:00 2026-02-22T22:54:45.730 INFO:teuthology.orchestra.run.trial082.stdout:(27/35): liboath-2.6.12-1.el9.x86_64.rpm 644 kB/s | 49 kB 00:00 2026-02-22T22:54:45.731 INFO:teuthology.orchestra.run.trial082.stdout:(28/35): gperftools-libs-2.9.1-3.el9.x86_64.rpm 1.2 MB/s | 308 kB 00:00 2026-02-22T22:54:45.786 INFO:teuthology.orchestra.run.trial082.stdout:(29/35): libunwind-1.6.2-1.el9.x86_64.rpm 1.2 MB/s | 67 kB 00:00 2026-02-22T22:54:45.839 INFO:teuthology.orchestra.run.trial082.stdout:(30/35): parquet-libs-9.0.0-15.el9.x86_64.rpm 7.6 MB/s | 838 kB 00:00 2026-02-22T22:54:45.875 INFO:teuthology.orchestra.run.trial082.stdout:(31/35): re2-20211101-20.el9.x86_64.rpm 2.1 MB/s | 191 kB 00:00 2026-02-22T22:54:45.898 INFO:teuthology.orchestra.run.trial082.stdout:(32/35): libarrow-9.0.0-15.el9.x86_64.rpm 12 MB/s | 4.4 MB 00:00 2026-02-22T22:54:45.939 INFO:teuthology.orchestra.run.trial082.stdout:(33/35): thrift-0.15.0-4.el9.x86_64.rpm 16 MB/s | 1.6 MB 00:00 2026-02-22T22:54:46.003 INFO:teuthology.orchestra.run.trial082.stdout:(34/35): librados2-19.2.3-689.gf90f4eae.el9.x86 27 MB/s | 3.4 MB 00:00 2026-02-22T22:54:46.042 INFO:teuthology.orchestra.run.trial082.stdout:(35/35): librbd1-19.2.3-689.gf90f4eae.el9.x86_6 22 MB/s | 3.2 MB 00:00 2026-02-22T22:54:46.045 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:54:46.045 INFO:teuthology.orchestra.run.trial082.stdout:Total 18 MB/s | 63 MB 00:03 2026-02-22T22:54:46.098 INFO:teuthology.orchestra.run.trial060.stdout:(35/35): nvme-cli-2.16-1.el9.x86_64.rpm 650 kB/s | 1.2 MB 00:01 2026-02-22T22:54:46.105 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:54:46.105 INFO:teuthology.orchestra.run.trial060.stdout:Total 18 MB/s | 63 MB 00:03 2026-02-22T22:54:46.148 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:54:46.166 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:54:46.166 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:54:46.204 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:54:46.222 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:54:46.222 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:54:46.422 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:54:46.422 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:54:46.478 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:54:46.478 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:54:46.714 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:54:46.724 INFO:teuthology.orchestra.run.trial082.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/37 2026-02-22T22:54:46.726 INFO:teuthology.orchestra.run.trial082.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/37 2026-02-22T22:54:46.738 INFO:teuthology.orchestra.run.trial082.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/37 2026-02-22T22:54:46.769 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:54:46.778 INFO:teuthology.orchestra.run.trial060.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/37 2026-02-22T22:54:46.781 INFO:teuthology.orchestra.run.trial060.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/37 2026-02-22T22:54:46.792 INFO:teuthology.orchestra.run.trial060.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/37 2026-02-22T22:54:46.890 INFO:teuthology.orchestra.run.trial082.stdout: Installing : thrift-0.15.0-4.el9.x86_64 4/37 2026-02-22T22:54:46.894 INFO:teuthology.orchestra.run.trial082.stdout: Upgrading : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:54:46.924 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:54:46.942 INFO:teuthology.orchestra.run.trial060.stdout: Installing : thrift-0.15.0-4.el9.x86_64 4/37 2026-02-22T22:54:46.948 INFO:teuthology.orchestra.run.trial060.stdout: Upgrading : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:54:46.967 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:54:46.968 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:54:46.975 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:54:46.981 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:54:46.985 INFO:teuthology.orchestra.run.trial082.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/37 2026-02-22T22:54:46.994 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 9/37 2026-02-22T22:54:47.016 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:54:47.017 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:54:47.020 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:54:47.021 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:54:47.031 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:54:47.032 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:54:47.036 INFO:teuthology.orchestra.run.trial060.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/37 2026-02-22T22:54:47.044 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 9/37 2026-02-22T22:54:47.065 INFO:teuthology.orchestra.run.trial082.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/37 2026-02-22T22:54:47.067 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:54:47.067 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:54:47.069 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 13/37 2026-02-22T22:54:47.075 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/37 2026-02-22T22:54:47.077 INFO:teuthology.orchestra.run.trial082.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/37 2026-02-22T22:54:47.080 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 16/37 2026-02-22T22:54:47.081 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:54:47.086 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/37 2026-02-22T22:54:47.111 INFO:teuthology.orchestra.run.trial060.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/37 2026-02-22T22:54:47.114 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 13/37 2026-02-22T22:54:47.122 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/37 2026-02-22T22:54:47.124 INFO:teuthology.orchestra.run.trial060.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/37 2026-02-22T22:54:47.127 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 16/37 2026-02-22T22:54:47.132 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/37 2026-02-22T22:54:47.225 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:54:47.226 INFO:teuthology.orchestra.run.trial082.stdout: Upgrading : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:54:47.252 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:54:47.256 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 20/37 2026-02-22T22:54:47.262 INFO:teuthology.orchestra.run.trial082.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/37 2026-02-22T22:54:47.272 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:54:47.273 INFO:teuthology.orchestra.run.trial060.stdout: Upgrading : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:54:47.299 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:54:47.303 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 20/37 2026-02-22T22:54:47.311 INFO:teuthology.orchestra.run.trial060.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/37 2026-02-22T22:54:47.487 INFO:teuthology.orchestra.run.trial082.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/37 2026-02-22T22:54:47.488 INFO:teuthology.orchestra.run.trial082.stdout: Installing : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:54:47.504 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:54:47.508 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 24/37 2026-02-22T22:54:47.513 INFO:teuthology.orchestra.run.trial082.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/37 2026-02-22T22:54:47.515 INFO:teuthology.orchestra.run.trial082.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/37 2026-02-22T22:54:47.516 INFO:teuthology.orchestra.run.trial082.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:54:47.534 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:54:47.536 INFO:teuthology.orchestra.run.trial060.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/37 2026-02-22T22:54:47.537 INFO:teuthology.orchestra.run.trial060.stdout: Installing : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:54:47.553 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:54:47.558 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 24/37 2026-02-22T22:54:47.562 INFO:teuthology.orchestra.run.trial060.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/37 2026-02-22T22:54:47.565 INFO:teuthology.orchestra.run.trial060.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/37 2026-02-22T22:54:47.566 INFO:teuthology.orchestra.run.trial060.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:54:47.583 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:54:47.648 INFO:teuthology.orchestra.run.trial082.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/37 2026-02-22T22:54:47.650 INFO:teuthology.orchestra.run.trial082.stdout: Installing : nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:54:47.696 INFO:teuthology.orchestra.run.trial060.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/37 2026-02-22T22:54:47.699 INFO:teuthology.orchestra.run.trial060.stdout: Installing : nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:54:47.934 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:54:47.934 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T22:54:47.934 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:47.970 INFO:teuthology.orchestra.run.trial082.stdout: Installing : mailcap-2.1.49-5.el9.noarch 30/37 2026-02-22T22:54:47.972 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 31/37 2026-02-22T22:54:47.992 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:54:47.992 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T22:54:47.992 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:48.027 INFO:teuthology.orchestra.run.trial060.stdout: Installing : mailcap-2.1.49-5.el9.noarch 30/37 2026-02-22T22:54:48.029 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 31/37 2026-02-22T22:54:48.950 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:54:48.952 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:54:49.011 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:54:49.014 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:54:49.233 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:54:49.236 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:54:49.274 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:54:49.274 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-22T22:54:49.274 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-22T22:54:49.274 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:49.278 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:54:49.296 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:54:49.300 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:54:49.341 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:54:49.341 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-22T22:54:49.341 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-22T22:54:49.341 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:49.344 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:54:53.785 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:54:53.785 INFO:teuthology.orchestra.run.trial082.stdout:skipping the directory /sys 2026-02-22T22:54:53.785 INFO:teuthology.orchestra.run.trial082.stdout:skipping the directory /proc 2026-02-22T22:54:53.785 INFO:teuthology.orchestra.run.trial082.stdout:skipping the directory /mnt 2026-02-22T22:54:53.785 INFO:teuthology.orchestra.run.trial082.stdout:skipping the directory /var/tmp 2026-02-22T22:54:53.786 INFO:teuthology.orchestra.run.trial082.stdout:skipping the directory /home 2026-02-22T22:54:53.786 INFO:teuthology.orchestra.run.trial082.stdout:skipping the directory /root 2026-02-22T22:54:53.786 INFO:teuthology.orchestra.run.trial082.stdout:skipping the directory /tmp 2026-02-22T22:54:53.786 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:53.857 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:54:53.857 INFO:teuthology.orchestra.run.trial060.stdout:skipping the directory /sys 2026-02-22T22:54:53.857 INFO:teuthology.orchestra.run.trial060.stdout:skipping the directory /proc 2026-02-22T22:54:53.858 INFO:teuthology.orchestra.run.trial060.stdout:skipping the directory /mnt 2026-02-22T22:54:53.858 INFO:teuthology.orchestra.run.trial060.stdout:skipping the directory /var/tmp 2026-02-22T22:54:53.858 INFO:teuthology.orchestra.run.trial060.stdout:skipping the directory /home 2026-02-22T22:54:53.858 INFO:teuthology.orchestra.run.trial060.stdout:skipping the directory /root 2026-02-22T22:54:53.858 INFO:teuthology.orchestra.run.trial060.stdout:skipping the directory /tmp 2026-02-22T22:54:53.858 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:54.241 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:54:54.265 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:54:54.266 INFO:teuthology.orchestra.run.trial082.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:54:54.266 INFO:teuthology.orchestra.run.trial082.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-22T22:54:54.266 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:54:54.266 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:54:54.266 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:54.267 INFO:teuthology.orchestra.run.trial082.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:54:54.280 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:54:54.281 INFO:teuthology.orchestra.run.trial082.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:54:54.309 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:54:54.332 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:54:54.332 INFO:teuthology.orchestra.run.trial060.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:54:54.333 INFO:teuthology.orchestra.run.trial060.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-22T22:54:54.333 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:54:54.333 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:54:54.333 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:54.334 INFO:teuthology.orchestra.run.trial060.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:54:54.336 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:54:54.336 INFO:teuthology.orchestra.run.trial082.stdout:Creating group 'qat' with GID 992. 2026-02-22T22:54:54.336 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:54.347 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:54:54.347 INFO:teuthology.orchestra.run.trial060.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:54:54.403 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:54:54.404 INFO:teuthology.orchestra.run.trial060.stdout:Creating group 'qat' with GID 992. 2026-02-22T22:54:54.404 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:54.633 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 1/37 2026-02-22T22:54:54.633 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2/37 2026-02-22T22:54:54.633 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 3/37 2026-02-22T22:54:54.633 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 4/37 2026-02-22T22:54:54.633 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:54:54.633 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:54:54.633 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:54:54.633 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 8/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 9/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 12/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 13/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : nvme-cli-2.16-1.el9.x86_64 15/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 16/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 17/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 19/37 2026-02-22T22:54:54.634 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 20/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 21/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 22/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 23/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 24/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 25/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 26/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 27/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 28/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 29/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 31/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : re2-1:20211101-20.el9.x86_64 32/37 2026-02-22T22:54:54.635 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 33/37 2026-02-22T22:54:54.636 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:54:54.636 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 35/37 2026-02-22T22:54:54.636 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 36/37 2026-02-22T22:54:54.693 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 1/37 2026-02-22T22:54:54.693 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 3/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 4/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 8/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 9/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 12/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 13/37 2026-02-22T22:54:54.694 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : nvme-cli-2.16-1.el9.x86_64 15/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 16/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 17/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 19/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 20/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 21/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 22/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 23/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 24/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 25/37 2026-02-22T22:54:54.695 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 26/37 2026-02-22T22:54:54.696 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 27/37 2026-02-22T22:54:54.696 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 28/37 2026-02-22T22:54:54.696 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 29/37 2026-02-22T22:54:54.696 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/37 2026-02-22T22:54:54.696 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 31/37 2026-02-22T22:54:54.696 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : re2-1:20211101-20.el9.x86_64 32/37 2026-02-22T22:54:54.696 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 33/37 2026-02-22T22:54:54.696 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:54:54.696 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 35/37 2026-02-22T22:54:54.696 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 36/37 2026-02-22T22:54:54.697 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:54:54.697 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:54.697 INFO:teuthology.orchestra.run.trial082.stdout:Upgraded: 2026-02-22T22:54:54.697 INFO:teuthology.orchestra.run.trial082.stdout: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.697 INFO:teuthology.orchestra.run.trial082.stdout: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.697 INFO:teuthology.orchestra.run.trial082.stdout:Installed: 2026-02-22T22:54:54.697 INFO:teuthology.orchestra.run.trial082.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-22T22:54:54.697 INFO:teuthology.orchestra.run.trial082.stdout: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.698 INFO:teuthology.orchestra.run.trial082.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: nvme-cli-2.16-1.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.699 INFO:teuthology.orchestra.run.trial082.stdout: python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.700 INFO:teuthology.orchestra.run.trial082.stdout: python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.700 INFO:teuthology.orchestra.run.trial082.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-22T22:54:54.700 INFO:teuthology.orchestra.run.trial082.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-22T22:54:54.700 INFO:teuthology.orchestra.run.trial082.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-22T22:54:54.700 INFO:teuthology.orchestra.run.trial082.stdout: re2-1:20211101-20.el9.x86_64 2026-02-22T22:54:54.700 INFO:teuthology.orchestra.run.trial082.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-22T22:54:54.700 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:54.700 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:54:54.735 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-test 2026-02-22T22:54:54.755 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:54:54.755 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout:Upgraded: 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout:Installed: 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-22T22:54:54.756 INFO:teuthology.orchestra.run.trial060.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-22T22:54:54.757 INFO:teuthology.orchestra.run.trial060.stdout: nvme-cli-2.16-1.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-22T22:54:54.758 INFO:teuthology.orchestra.run.trial060.stdout: re2-1:20211101-20.el9.x86_64 2026-02-22T22:54:54.759 INFO:teuthology.orchestra.run.trial060.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-22T22:54:54.759 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:54.759 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:54:54.797 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-test 2026-02-22T22:54:55.043 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:14 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:54:55.107 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:14 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:54:55.126 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:54:55.127 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:54:55.128 INFO:teuthology.orchestra.run.trial082.stdout: Package Arch Version Repository Size 2026-02-22T22:54:55.128 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:54:55.128 INFO:teuthology.orchestra.run.trial082.stdout:Installing: 2026-02-22T22:54:55.128 INFO:teuthology.orchestra.run.trial082.stdout: ceph-test x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 50 M 2026-02-22T22:54:55.128 INFO:teuthology.orchestra.run.trial082.stdout:Installing dependencies: 2026-02-22T22:54:55.128 INFO:teuthology.orchestra.run.trial082.stdout: libcephsqlite x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 163 k 2026-02-22T22:54:55.128 INFO:teuthology.orchestra.run.trial082.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-22T22:54:55.128 INFO:teuthology.orchestra.run.trial082.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-22T22:54:55.128 INFO:teuthology.orchestra.run.trial082.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-22T22:54:55.128 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:55.129 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:54:55.129 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:54:55.129 INFO:teuthology.orchestra.run.trial082.stdout:Install 5 Packages 2026-02-22T22:54:55.129 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:55.129 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 50 M 2026-02-22T22:54:55.129 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 212 M 2026-02-22T22:54:55.129 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:54:55.191 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:54:55.191 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:54:55.192 INFO:teuthology.orchestra.run.trial060.stdout: Package Arch Version Repository Size 2026-02-22T22:54:55.192 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:54:55.192 INFO:teuthology.orchestra.run.trial060.stdout:Installing: 2026-02-22T22:54:55.192 INFO:teuthology.orchestra.run.trial060.stdout: ceph-test x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 50 M 2026-02-22T22:54:55.192 INFO:teuthology.orchestra.run.trial060.stdout:Installing dependencies: 2026-02-22T22:54:55.192 INFO:teuthology.orchestra.run.trial060.stdout: libcephsqlite x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 163 k 2026-02-22T22:54:55.192 INFO:teuthology.orchestra.run.trial060.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-22T22:54:55.192 INFO:teuthology.orchestra.run.trial060.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-22T22:54:55.192 INFO:teuthology.orchestra.run.trial060.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-22T22:54:55.192 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:55.193 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:54:55.193 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:54:55.193 INFO:teuthology.orchestra.run.trial060.stdout:Install 5 Packages 2026-02-22T22:54:55.193 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:54:55.193 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 50 M 2026-02-22T22:54:55.193 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 212 M 2026-02-22T22:54:55.193 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:54:55.276 INFO:teuthology.orchestra.run.trial082.stdout:(1/5): libcephsqlite-19.2.3-689.gf90f4eae.el9.x 1.8 MB/s | 163 kB 00:00 2026-02-22T22:54:55.321 INFO:teuthology.orchestra.run.trial060.stdout:(1/5): libxslt-1.1.34-12.el9.x86_64.rpm 3.2 MB/s | 233 kB 00:00 2026-02-22T22:54:55.341 INFO:teuthology.orchestra.run.trial060.stdout:(2/5): socat-1.7.4.1-8.el9.x86_64.rpm 15 MB/s | 303 kB 00:00 2026-02-22T22:54:55.351 INFO:teuthology.orchestra.run.trial060.stdout:(3/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 6.6 MB/s | 64 kB 00:00 2026-02-22T22:54:55.385 INFO:teuthology.orchestra.run.trial060.stdout:(4/5): libcephsqlite-19.2.3-689.gf90f4eae.el9.x 1.2 MB/s | 163 kB 00:00 2026-02-22T22:54:55.526 INFO:teuthology.orchestra.run.trial082.stdout:(2/5): libxslt-1.1.34-12.el9.x86_64.rpm 689 kB/s | 233 kB 00:00 2026-02-22T22:54:55.606 INFO:teuthology.orchestra.run.trial082.stdout:(3/5): socat-1.7.4.1-8.el9.x86_64.rpm 917 kB/s | 303 kB 00:00 2026-02-22T22:54:55.756 INFO:teuthology.orchestra.run.trial082.stdout:(4/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 277 kB/s | 64 kB 00:00 2026-02-22T22:54:56.236 INFO:teuthology.orchestra.run.trial082.stdout:(5/5): ceph-test-19.2.3-689.gf90f4eae.el9.x86_6 47 MB/s | 50 MB 00:01 2026-02-22T22:54:56.237 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:54:56.237 INFO:teuthology.orchestra.run.trial082.stdout:Total 45 MB/s | 50 MB 00:01 2026-02-22T22:54:56.264 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:54:56.273 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:54:56.273 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:54:56.459 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:54:56.459 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:54:56.671 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:54:56.676 INFO:teuthology.orchestra.run.trial082.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-22T22:54:56.681 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-22T22:54:56.689 INFO:teuthology.orchestra.run.trial082.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-22T22:54:56.690 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:54:57.177 INFO:teuthology.orchestra.run.trial060.stdout:(5/5): ceph-test-19.2.3-689.gf90f4eae.el9.x86_6 26 MB/s | 50 MB 00:01 2026-02-22T22:54:57.179 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:54:57.179 INFO:teuthology.orchestra.run.trial060.stdout:Total 25 MB/s | 50 MB 00:01 2026-02-22T22:54:57.206 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:54:57.215 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:54:57.215 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:54:57.402 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:54:57.402 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:54:57.613 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:54:57.618 INFO:teuthology.orchestra.run.trial060.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-22T22:54:57.622 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-22T22:54:57.630 INFO:teuthology.orchestra.run.trial060.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-22T22:54:57.631 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:54:58.978 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:54:58.996 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:54:59.220 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:54:59.220 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 1/5 2026-02-22T22:54:59.220 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2/5 2026-02-22T22:54:59.220 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-22T22:54:59.220 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-22T22:54:59.275 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-22T22:54:59.275 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:59.275 INFO:teuthology.orchestra.run.trial082.stdout:Installed: 2026-02-22T22:54:59.276 INFO:teuthology.orchestra.run.trial082.stdout: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:59.276 INFO:teuthology.orchestra.run.trial082.stdout: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:54:59.276 INFO:teuthology.orchestra.run.trial082.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-22T22:54:59.276 INFO:teuthology.orchestra.run.trial082.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-22T22:54:59.276 INFO:teuthology.orchestra.run.trial082.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-22T22:54:59.276 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:59.276 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:54:59.308 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph 2026-02-22T22:54:59.615 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:18 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:54:59.702 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:54:59.705 INFO:teuthology.orchestra.run.trial082.stdout:====================================================================================== 2026-02-22T22:54:59.705 INFO:teuthology.orchestra.run.trial082.stdout: Package Arch Version Repository Size 2026-02-22T22:54:59.705 INFO:teuthology.orchestra.run.trial082.stdout:====================================================================================== 2026-02-22T22:54:59.705 INFO:teuthology.orchestra.run.trial082.stdout:Installing: 2026-02-22T22:54:59.705 INFO:teuthology.orchestra.run.trial082.stdout: ceph x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 6.5 k 2026-02-22T22:54:59.705 INFO:teuthology.orchestra.run.trial082.stdout:Installing dependencies: 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: ceph-grafana-dashboards noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 31 k 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mds x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 2.4 M 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.1 M 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-modules-core noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 253 k 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mon x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 4.7 M 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: ceph-osd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 17 M 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: ceph-prometheus-alerts noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 17 k 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 769 k 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-22T22:54:59.706 INFO:teuthology.orchestra.run.trial082.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-22T22:54:59.707 INFO:teuthology.orchestra.run.trial082.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-22T22:54:59.708 INFO:teuthology.orchestra.run.trial082.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-22T22:54:59.708 INFO:teuthology.orchestra.run.trial082.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-22T22:54:59.708 INFO:teuthology.orchestra.run.trial082.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-22T22:54:59.708 INFO:teuthology.orchestra.run.trial082.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-22T22:54:59.708 INFO:teuthology.orchestra.run.trial082.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-22T22:54:59.708 INFO:teuthology.orchestra.run.trial082.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-22T22:54:59.708 INFO:teuthology.orchestra.run.trial082.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-22T22:54:59.708 INFO:teuthology.orchestra.run.trial082.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-22T22:54:59.708 INFO:teuthology.orchestra.run.trial082.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-22T22:54:59.708 INFO:teuthology.orchestra.run.trial082.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-22T22:54:59.709 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-22T22:54:59.710 INFO:teuthology.orchestra.run.trial082.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-22T22:54:59.711 INFO:teuthology.orchestra.run.trial082.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-22T22:54:59.712 INFO:teuthology.orchestra.run.trial082.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-22T22:54:59.713 INFO:teuthology.orchestra.run.trial082.stdout:Installing weak dependencies: 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 150 k 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-dashboard noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 3.8 M 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-diskprediction-local noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 7.4 M 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-k8sevents noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 22 k 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-rook noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 49 k 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout: ceph-volume noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 299 k 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout:====================================================================================== 2026-02-22T22:54:59.714 INFO:teuthology.orchestra.run.trial082.stdout:Install 99 Packages 2026-02-22T22:54:59.715 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:54:59.715 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 95 M 2026-02-22T22:54:59.716 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 520 M 2026-02-22T22:54:59.716 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:54:59.880 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:54:59.898 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:55:00.107 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:55:00.107 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 1/5 2026-02-22T22:55:00.108 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2/5 2026-02-22T22:55:00.108 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-22T22:55:00.108 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-22T22:55:00.161 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-22T22:55:00.162 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:00.162 INFO:teuthology.orchestra.run.trial060.stdout:Installed: 2026-02-22T22:55:00.162 INFO:teuthology.orchestra.run.trial060.stdout: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:00.162 INFO:teuthology.orchestra.run.trial060.stdout: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:00.162 INFO:teuthology.orchestra.run.trial060.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-22T22:55:00.162 INFO:teuthology.orchestra.run.trial060.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-22T22:55:00.162 INFO:teuthology.orchestra.run.trial060.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-22T22:55:00.163 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:00.163 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:00.195 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph 2026-02-22T22:55:00.499 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:19 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:00.582 INFO:teuthology.orchestra.run.trial082.stdout:(1/99): ceph-19.2.3-689.gf90f4eae.el9.x86_64.rp 125 kB/s | 6.5 kB 00:00 2026-02-22T22:55:00.585 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:00.588 INFO:teuthology.orchestra.run.trial060.stdout:====================================================================================== 2026-02-22T22:55:00.588 INFO:teuthology.orchestra.run.trial060.stdout: Package Arch Version Repository Size 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout:====================================================================================== 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout:Installing: 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout: ceph x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 6.5 k 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout:Installing dependencies: 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout: ceph-grafana-dashboards noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 31 k 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mds x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 2.4 M 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.1 M 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-modules-core noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 253 k 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mon x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 4.7 M 2026-02-22T22:55:00.589 INFO:teuthology.orchestra.run.trial060.stdout: ceph-osd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 17 M 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: ceph-prometheus-alerts noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 17 k 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 769 k 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-22T22:55:00.590 INFO:teuthology.orchestra.run.trial060.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-22T22:55:00.591 INFO:teuthology.orchestra.run.trial060.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-22T22:55:00.592 INFO:teuthology.orchestra.run.trial060.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2026-02-22T22:55:00.593 INFO:teuthology.orchestra.run.trial060.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-22T22:55:00.594 INFO:teuthology.orchestra.run.trial060.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-22T22:55:00.595 INFO:teuthology.orchestra.run.trial060.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-22T22:55:00.596 INFO:teuthology.orchestra.run.trial060.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout:Installing weak dependencies: 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 150 k 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-dashboard noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 3.8 M 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-diskprediction-local noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 7.4 M 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-k8sevents noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 22 k 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-rook noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 49 k 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout: ceph-volume noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 299 k 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:00.597 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:55:00.598 INFO:teuthology.orchestra.run.trial060.stdout:====================================================================================== 2026-02-22T22:55:00.598 INFO:teuthology.orchestra.run.trial060.stdout:Install 99 Packages 2026-02-22T22:55:00.598 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:00.598 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 95 M 2026-02-22T22:55:00.599 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 520 M 2026-02-22T22:55:00.599 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:55:00.673 INFO:teuthology.orchestra.run.trial082.stdout:(2/99): ceph-mgr-19.2.3-689.gf90f4eae.el9.x86_6 7.5 MB/s | 1.1 MB 00:00 2026-02-22T22:55:00.700 INFO:teuthology.orchestra.run.trial082.stdout:(3/99): ceph-mds-19.2.3-689.gf90f4eae.el9.x86_6 14 MB/s | 2.4 MB 00:00 2026-02-22T22:55:00.736 INFO:teuthology.orchestra.run.trial082.stdout:(4/99): ceph-grafana-dashboards-19.2.3-689.gf90 866 kB/s | 31 kB 00:00 2026-02-22T22:55:00.774 INFO:teuthology.orchestra.run.trial082.stdout:(5/99): ceph-mgr-cephadm-19.2.3-689.gf90f4eae.e 3.9 MB/s | 150 kB 00:00 2026-02-22T22:55:00.855 INFO:teuthology.orchestra.run.trial082.stdout:(6/99): ceph-mon-19.2.3-689.gf90f4eae.el9.x86_6 17 MB/s | 4.7 MB 00:00 2026-02-22T22:55:00.943 INFO:teuthology.orchestra.run.trial082.stdout:(7/99): ceph-mgr-dashboard-19.2.3-689.gf90f4eae 23 MB/s | 3.8 MB 00:00 2026-02-22T22:55:00.988 INFO:teuthology.orchestra.run.trial082.stdout:(8/99): ceph-mgr-k8sevents-19.2.3-689.gf90f4eae 486 kB/s | 22 kB 00:00 2026-02-22T22:55:01.040 INFO:teuthology.orchestra.run.trial082.stdout:(9/99): ceph-mgr-modules-core-19.2.3-689.gf90f4 4.8 MB/s | 253 kB 00:00 2026-02-22T22:55:01.087 INFO:teuthology.orchestra.run.trial060.stdout:(1/99): ceph-19.2.3-689.gf90f4eae.el9.x86_64.rp 31 kB/s | 6.5 kB 00:00 2026-02-22T22:55:01.090 INFO:teuthology.orchestra.run.trial082.stdout:(10/99): ceph-mgr-rook-19.2.3-689.gf90f4eae.el9 991 kB/s | 49 kB 00:00 2026-02-22T22:55:01.141 INFO:teuthology.orchestra.run.trial082.stdout:(11/99): ceph-prometheus-alerts-19.2.3-689.gf90 325 kB/s | 17 kB 00:00 2026-02-22T22:55:01.201 INFO:teuthology.orchestra.run.trial082.stdout:(12/99): ceph-volume-19.2.3-689.gf90f4eae.el9.n 4.9 MB/s | 299 kB 00:00 2026-02-22T22:55:01.223 INFO:teuthology.orchestra.run.trial082.stdout:(13/99): ceph-mgr-diskprediction-local-19.2.3-6 20 MB/s | 7.4 MB 00:00 2026-02-22T22:55:01.256 INFO:teuthology.orchestra.run.trial082.stdout:(14/99): cephadm-19.2.3-689.gf90f4eae.el9.noarc 14 MB/s | 769 kB 00:00 2026-02-22T22:55:01.293 INFO:teuthology.orchestra.run.trial082.stdout:(15/99): ceph-osd-19.2.3-689.gf90f4eae.el9.x86_ 28 MB/s | 17 MB 00:00 2026-02-22T22:55:01.294 INFO:teuthology.orchestra.run.trial082.stdout:(16/99): ledmon-libs-1.1.0-3.el9.x86_64.rpm 1.0 MB/s | 40 kB 00:00 2026-02-22T22:55:01.324 INFO:teuthology.orchestra.run.trial082.stdout:(17/99): cryptsetup-2.8.1-3.el9.x86_64.rpm 3.4 MB/s | 351 kB 00:00 2026-02-22T22:55:01.335 INFO:teuthology.orchestra.run.trial082.stdout:(18/99): libconfig-1.7.2-9.el9.x86_64.rpm 1.7 MB/s | 72 kB 00:00 2026-02-22T22:55:01.337 INFO:teuthology.orchestra.run.trial082.stdout:(19/99): libquadmath-11.5.0-14.el9.x86_64.rpm 14 MB/s | 184 kB 00:00 2026-02-22T22:55:01.342 INFO:teuthology.orchestra.run.trial082.stdout:(20/99): libgfortran-11.5.0-14.el9.x86_64.rpm 16 MB/s | 794 kB 00:00 2026-02-22T22:55:01.353 INFO:teuthology.orchestra.run.trial082.stdout:(21/99): python3-ply-3.11-14.el9.noarch.rpm 9.6 MB/s | 106 kB 00:00 2026-02-22T22:55:01.362 INFO:teuthology.orchestra.run.trial060.stdout:(2/99): ceph-mgr-19.2.3-689.gf90f4eae.el9.x86_6 2.2 MB/s | 1.1 MB 00:00 2026-02-22T22:55:01.363 INFO:teuthology.orchestra.run.trial082.stdout:(22/99): python3-pycparser-2.20-6.el9.noarch.rp 13 MB/s | 135 kB 00:00 2026-02-22T22:55:01.366 INFO:teuthology.orchestra.run.trial082.stdout:(23/99): python3-cffi-1.14.5-5.el9.x86_64.rpm 7.9 MB/s | 253 kB 00:00 2026-02-22T22:55:01.374 INFO:teuthology.orchestra.run.trial082.stdout:(24/99): python3-pyparsing-2.4.7-9.el9.noarch.r 13 MB/s | 150 kB 00:00 2026-02-22T22:55:01.376 INFO:teuthology.orchestra.run.trial082.stdout:(25/99): python3-cryptography-36.0.1-5.el9.x86_ 32 MB/s | 1.2 MB 00:00 2026-02-22T22:55:01.379 INFO:teuthology.orchestra.run.trial082.stdout:(26/99): python3-requests-2.25.1-10.el9.noarch. 9.6 MB/s | 126 kB 00:00 2026-02-22T22:55:01.386 INFO:teuthology.orchestra.run.trial082.stdout:(27/99): python3-urllib3-1.26.5-7.el9.noarch.rp 18 MB/s | 218 kB 00:00 2026-02-22T22:55:01.388 INFO:teuthology.orchestra.run.trial082.stdout:(28/99): unzip-6.0-59.el9.x86_64.rpm 15 MB/s | 182 kB 00:00 2026-02-22T22:55:01.395 INFO:teuthology.orchestra.run.trial082.stdout:(29/99): zip-3.0-35.el9.x86_64.rpm 16 MB/s | 266 kB 00:00 2026-02-22T22:55:01.467 INFO:teuthology.orchestra.run.trial060.stdout:(3/99): ceph-mds-19.2.3-689.gf90f4eae.el9.x86_6 4.1 MB/s | 2.4 MB 00:00 2026-02-22T22:55:01.479 INFO:teuthology.orchestra.run.trial060.stdout:(4/99): ceph-grafana-dashboards-19.2.3-689.gf90 2.5 MB/s | 31 kB 00:00 2026-02-22T22:55:01.491 INFO:teuthology.orchestra.run.trial060.stdout:(5/99): ceph-mgr-cephadm-19.2.3-689.gf90f4eae.e 12 MB/s | 150 kB 00:00 2026-02-22T22:55:01.676 INFO:teuthology.orchestra.run.trial060.stdout:(6/99): ceph-mon-19.2.3-689.gf90f4eae.el9.x86_6 8.0 MB/s | 4.7 MB 00:00 2026-02-22T22:55:01.681 INFO:teuthology.orchestra.run.trial060.stdout:(7/99): ceph-mgr-dashboard-19.2.3-689.gf90f4eae 20 MB/s | 3.8 MB 00:00 2026-02-22T22:55:01.695 INFO:teuthology.orchestra.run.trial060.stdout:(8/99): ceph-mgr-k8sevents-19.2.3-689.gf90f4eae 1.5 MB/s | 22 kB 00:00 2026-02-22T22:55:01.713 INFO:teuthology.orchestra.run.trial060.stdout:(9/99): ceph-mgr-modules-core-19.2.3-689.gf90f4 14 MB/s | 253 kB 00:00 2026-02-22T22:55:01.728 INFO:teuthology.orchestra.run.trial060.stdout:(10/99): ceph-mgr-rook-19.2.3-689.gf90f4eae.el9 3.2 MB/s | 49 kB 00:00 2026-02-22T22:55:01.743 INFO:teuthology.orchestra.run.trial060.stdout:(11/99): ceph-prometheus-alerts-19.2.3-689.gf90 1.1 MB/s | 17 kB 00:00 2026-02-22T22:55:01.765 INFO:teuthology.orchestra.run.trial060.stdout:(12/99): ceph-volume-19.2.3-689.gf90f4eae.el9.n 14 MB/s | 299 kB 00:00 2026-02-22T22:55:01.813 INFO:teuthology.orchestra.run.trial060.stdout:(13/99): cephadm-19.2.3-689.gf90f4eae.el9.noarc 16 MB/s | 769 kB 00:00 2026-02-22T22:55:01.929 INFO:teuthology.orchestra.run.trial060.stdout:(14/99): cryptsetup-2.8.1-3.el9.x86_64.rpm 2.9 MB/s | 351 kB 00:00 2026-02-22T22:55:01.941 INFO:teuthology.orchestra.run.trial082.stdout:(30/99): flexiblas-3.0.4-9.el9.x86_64.rpm 53 kB/s | 30 kB 00:00 2026-02-22T22:55:01.948 INFO:teuthology.orchestra.run.trial082.stdout:(31/99): flexiblas-openblas-openmp-3.0.4-9.el9. 27 kB/s | 15 kB 00:00 2026-02-22T22:55:01.952 INFO:teuthology.orchestra.run.trial060.stdout:(15/99): ledmon-libs-1.1.0-3.el9.x86_64.rpm 1.8 MB/s | 40 kB 00:00 2026-02-22T22:55:01.974 INFO:teuthology.orchestra.run.trial060.stdout:(16/99): libconfig-1.7.2-9.el9.x86_64.rpm 3.1 MB/s | 72 kB 00:00 2026-02-22T22:55:02.024 INFO:teuthology.orchestra.run.trial060.stdout:(17/99): ceph-osd-19.2.3-689.gf90f4eae.el9.x86_ 26 MB/s | 17 MB 00:00 2026-02-22T22:55:02.026 INFO:teuthology.orchestra.run.trial060.stdout:(18/99): libgfortran-11.5.0-14.el9.x86_64.rpm 15 MB/s | 794 kB 00:00 2026-02-22T22:55:02.045 INFO:teuthology.orchestra.run.trial060.stdout:(19/99): ceph-mgr-diskprediction-local-19.2.3-6 20 MB/s | 7.4 MB 00:00 2026-02-22T22:55:02.054 INFO:teuthology.orchestra.run.trial060.stdout:(20/99): python3-cffi-1.14.5-5.el9.x86_64.rpm 8.9 MB/s | 253 kB 00:00 2026-02-22T22:55:02.078 INFO:teuthology.orchestra.run.trial060.stdout:(21/99): python3-ply-3.11-14.el9.noarch.rpm 4.4 MB/s | 106 kB 00:00 2026-02-22T22:55:02.102 INFO:teuthology.orchestra.run.trial060.stdout:(22/99): python3-pycparser-2.20-6.el9.noarch.rp 5.4 MB/s | 135 kB 00:00 2026-02-22T22:55:02.109 INFO:teuthology.orchestra.run.trial060.stdout:(23/99): libquadmath-11.5.0-14.el9.x86_64.rpm 2.1 MB/s | 184 kB 00:00 2026-02-22T22:55:02.127 INFO:teuthology.orchestra.run.trial060.stdout:(24/99): python3-pyparsing-2.4.7-9.el9.noarch.r 6.0 MB/s | 150 kB 00:00 2026-02-22T22:55:02.134 INFO:teuthology.orchestra.run.trial060.stdout:(25/99): python3-requests-2.25.1-10.el9.noarch. 5.1 MB/s | 126 kB 00:00 2026-02-22T22:55:02.154 INFO:teuthology.orchestra.run.trial060.stdout:(26/99): python3-urllib3-1.26.5-7.el9.noarch.rp 8.0 MB/s | 218 kB 00:00 2026-02-22T22:55:02.157 INFO:teuthology.orchestra.run.trial060.stdout:(27/99): unzip-6.0-59.el9.x86_64.rpm 7.5 MB/s | 182 kB 00:00 2026-02-22T22:55:02.178 INFO:teuthology.orchestra.run.trial060.stdout:(28/99): zip-3.0-35.el9.x86_64.rpm 11 MB/s | 266 kB 00:00 2026-02-22T22:55:02.193 INFO:teuthology.orchestra.run.trial060.stdout:(29/99): python3-cryptography-36.0.1-5.el9.x86_ 8.4 MB/s | 1.2 MB 00:00 2026-02-22T22:55:02.332 INFO:teuthology.orchestra.run.trial082.stdout:(32/99): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 630 kB/s | 246 kB 00:00 2026-02-22T22:55:02.338 INFO:teuthology.orchestra.run.trial082.stdout:(33/99): lua-5.4.4-4.el9.x86_64.rpm 483 kB/s | 188 kB 00:00 2026-02-22T22:55:02.425 INFO:teuthology.orchestra.run.trial060.stdout:(30/99): flexiblas-3.0.4-9.el9.x86_64.rpm 111 kB/s | 30 kB 00:00 2026-02-22T22:55:02.427 INFO:teuthology.orchestra.run.trial060.stdout:(31/99): flexiblas-openblas-openmp-3.0.4-9.el9. 64 kB/s | 15 kB 00:00 2026-02-22T22:55:02.489 INFO:teuthology.orchestra.run.trial082.stdout:(34/99): openblas-0.3.29-1.el9.x86_64.rpm 268 kB/s | 42 kB 00:00 2026-02-22T22:55:02.746 INFO:teuthology.orchestra.run.trial060.stdout:(32/99): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 767 kB/s | 246 kB 00:00 2026-02-22T22:55:02.754 INFO:teuthology.orchestra.run.trial060.stdout:(33/99): lua-5.4.4-4.el9.x86_64.rpm 576 kB/s | 188 kB 00:00 2026-02-22T22:55:02.831 INFO:teuthology.orchestra.run.trial060.stdout:(34/99): openblas-0.3.29-1.el9.x86_64.rpm 496 kB/s | 42 kB 00:00 2026-02-22T22:55:02.957 INFO:teuthology.orchestra.run.trial060.stdout:(35/99): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 3.8 MB/s | 3.0 MB 00:00 2026-02-22T22:55:03.088 INFO:teuthology.orchestra.run.trial060.stdout:(36/99): protobuf-3.14.0-17.el9.x86_64.rpm 3.9 MB/s | 1.0 MB 00:00 2026-02-22T22:55:03.175 INFO:teuthology.orchestra.run.trial060.stdout:(37/99): python3-babel-2.9.1-2.el9.noarch.rpm 27 MB/s | 6.0 MB 00:00 2026-02-22T22:55:03.181 INFO:teuthology.orchestra.run.trial060.stdout:(38/99): python3-devel-3.9.25-3.el9.x86_64.rpm 2.6 MB/s | 244 kB 00:00 2026-02-22T22:55:03.233 INFO:teuthology.orchestra.run.trial060.stdout:(39/99): openblas-openmp-0.3.29-1.el9.x86_64.rp 11 MB/s | 5.3 MB 00:00 2026-02-22T22:55:03.256 INFO:teuthology.orchestra.run.trial060.stdout:(40/99): python3-jinja2-2.11.3-8.el9.noarch.rpm 3.0 MB/s | 249 kB 00:00 2026-02-22T22:55:03.266 INFO:teuthology.orchestra.run.trial060.stdout:(41/99): python3-libstoragemgmt-1.10.1-1.el9.x8 2.0 MB/s | 177 kB 00:00 2026-02-22T22:55:03.325 INFO:teuthology.orchestra.run.trial060.stdout:(42/99): python3-lxml-4.6.5-3.el9.x86_64.rpm 13 MB/s | 1.2 MB 00:00 2026-02-22T22:55:03.337 INFO:teuthology.orchestra.run.trial060.stdout:(43/99): python3-mako-1.1.4-6.el9.noarch.rpm 2.1 MB/s | 172 kB 00:00 2026-02-22T22:55:03.345 INFO:teuthology.orchestra.run.trial060.stdout:(44/99): python3-markupsafe-1.1.1-12.el9.x86_64 441 kB/s | 35 kB 00:00 2026-02-22T22:55:03.418 INFO:teuthology.orchestra.run.trial060.stdout:(45/99): python3-numpy-f2py-1.23.5-2.el9.x86_64 5.3 MB/s | 442 kB 00:00 2026-02-22T22:55:03.425 INFO:teuthology.orchestra.run.trial060.stdout:(46/99): python3-packaging-20.9-5.el9.noarch.rp 958 kB/s | 77 kB 00:00 2026-02-22T22:55:03.499 INFO:teuthology.orchestra.run.trial060.stdout:(47/99): python3-protobuf-3.14.0-17.el9.noarch. 3.2 MB/s | 267 kB 00:00 2026-02-22T22:55:03.510 INFO:teuthology.orchestra.run.trial060.stdout:(48/99): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.8 MB/s | 159 kB 00:00 2026-02-22T22:55:03.522 INFO:teuthology.orchestra.run.trial060.stdout:(49/99): python3-numpy-1.23.5-2.el9.x86_64.rpm 31 MB/s | 6.1 MB 00:00 2026-02-22T22:55:03.581 INFO:teuthology.orchestra.run.trial060.stdout:(50/99): python3-pyasn1-modules-0.4.8-6.el9.noa 3.4 MB/s | 279 kB 00:00 2026-02-22T22:55:03.590 INFO:teuthology.orchestra.run.trial060.stdout:(51/99): python3-requests-oauthlib-1.3.0-12.el9 674 kB/s | 54 kB 00:00 2026-02-22T22:55:03.660 INFO:teuthology.orchestra.run.trial060.stdout:(52/99): python3-toml-0.10.2-6.el9.noarch.rpm 528 kB/s | 42 kB 00:00 2026-02-22T22:55:03.675 INFO:teuthology.orchestra.run.trial060.stdout:(53/99): xmlsec1-1.2.29-13.el9.x86_64.rpm 2.2 MB/s | 189 kB 00:00 2026-02-22T22:55:03.738 INFO:teuthology.orchestra.run.trial082.stdout:(35/99): protobuf-3.14.0-17.el9.x86_64.rpm 824 kB/s | 1.0 MB 00:01 2026-02-22T22:55:03.739 INFO:teuthology.orchestra.run.trial060.stdout:(54/99): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 1.1 MB/s | 90 kB 00:00 2026-02-22T22:55:03.753 INFO:teuthology.orchestra.run.trial060.stdout:(55/99): lua-devel-5.4.4-4.el9.x86_64.rpm 286 kB/s | 22 kB 00:00 2026-02-22T22:55:03.825 INFO:teuthology.orchestra.run.trial060.stdout:(56/99): protobuf-compiler-3.14.0-17.el9.x86_64 9.9 MB/s | 862 kB 00:00 2026-02-22T22:55:03.936 INFO:teuthology.orchestra.run.trial060.stdout:(57/99): grpc-data-1.46.7-10.el9.noarch.rpm 174 kB/s | 19 kB 00:00 2026-02-22T22:55:03.970 INFO:teuthology.orchestra.run.trial060.stdout:(58/99): abseil-cpp-20211102.0-4.el9.x86_64.rpm 2.5 MB/s | 551 kB 00:00 2026-02-22T22:55:03.977 INFO:teuthology.orchestra.run.trial060.stdout:(59/99): luarocks-3.9.2-5.el9.noarch.rpm 3.6 MB/s | 151 kB 00:00 2026-02-22T22:55:04.007 INFO:teuthology.orchestra.run.trial060.stdout:(60/99): python3-asyncssh-2.13.2-5.el9.noarch.r 14 MB/s | 548 kB 00:00 2026-02-22T22:55:04.008 INFO:teuthology.orchestra.run.trial060.stdout:(61/99): python3-autocommand-2.2.2-8.el9.noarch 972 kB/s | 29 kB 00:00 2026-02-22T22:55:04.032 INFO:teuthology.orchestra.run.trial060.stdout:(62/99): python3-backports-tarfile-1.2.0-1.el9. 2.4 MB/s | 60 kB 00:00 2026-02-22T22:55:04.033 INFO:teuthology.orchestra.run.trial060.stdout:(63/99): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.7 MB/s | 43 kB 00:00 2026-02-22T22:55:04.057 INFO:teuthology.orchestra.run.trial060.stdout:(64/99): python3-cachetools-4.2.4-1.el9.noarch. 1.3 MB/s | 32 kB 00:00 2026-02-22T22:55:04.058 INFO:teuthology.orchestra.run.trial060.stdout:(65/99): python3-certifi-2023.05.07-4.el9.noarc 582 kB/s | 14 kB 00:00 2026-02-22T22:55:04.085 INFO:teuthology.orchestra.run.trial060.stdout:(66/99): python3-cheroot-10.0.1-4.el9.noarch.rp 6.2 MB/s | 173 kB 00:00 2026-02-22T22:55:04.112 INFO:teuthology.orchestra.run.trial060.stdout:(67/99): python3-scipy-1.9.3-2.el9.x86_64.rpm 33 MB/s | 19 MB 00:00 2026-02-22T22:55:04.113 INFO:teuthology.orchestra.run.trial060.stdout:(68/99): python3-cherrypy-18.6.1-2.el9.noarch.r 6.3 MB/s | 358 kB 00:00 2026-02-22T22:55:04.113 INFO:teuthology.orchestra.run.trial060.stdout:(69/99): python3-google-auth-2.45.0-1.el9.noarc 8.7 MB/s | 254 kB 00:00 2026-02-22T22:55:04.177 INFO:teuthology.orchestra.run.trial060.stdout:(70/99): python3-grpcio-1.46.7-10.el9.x86_64.rp 32 MB/s | 2.0 MB 00:00 2026-02-22T22:55:04.177 INFO:teuthology.orchestra.run.trial060.stdout:(71/99): python3-grpcio-tools-1.46.7-10.el9.x86 2.2 MB/s | 144 kB 00:00 2026-02-22T22:55:04.178 INFO:teuthology.orchestra.run.trial060.stdout:(72/99): python3-influxdb-5.3.1-1.el9.noarch.rp 2.1 MB/s | 139 kB 00:00 2026-02-22T22:55:04.201 INFO:teuthology.orchestra.run.trial060.stdout:(73/99): python3-isodate-0.6.1-3.el9.noarch.rpm 2.2 MB/s | 56 kB 00:00 2026-02-22T22:55:04.202 INFO:teuthology.orchestra.run.trial060.stdout:(74/99): python3-jaraco-8.2.1-3.el9.noarch.rpm 436 kB/s | 11 kB 00:00 2026-02-22T22:55:04.203 INFO:teuthology.orchestra.run.trial060.stdout:(75/99): python3-jaraco-classes-3.2.1-5.el9.noa 726 kB/s | 18 kB 00:00 2026-02-22T22:55:04.226 INFO:teuthology.orchestra.run.trial060.stdout:(76/99): python3-jaraco-collections-3.0.0-8.el9 948 kB/s | 23 kB 00:00 2026-02-22T22:55:04.227 INFO:teuthology.orchestra.run.trial060.stdout:(77/99): python3-jaraco-context-6.0.1-3.el9.noa 800 kB/s | 20 kB 00:00 2026-02-22T22:55:04.227 INFO:teuthology.orchestra.run.trial060.stdout:(78/99): python3-jaraco-functools-3.5.0-2.el9.n 801 kB/s | 19 kB 00:00 2026-02-22T22:55:04.250 INFO:teuthology.orchestra.run.trial060.stdout:(79/99): python3-jaraco-text-4.0.0-2.el9.noarch 1.1 MB/s | 26 kB 00:00 2026-02-22T22:55:04.258 INFO:teuthology.orchestra.run.trial060.stdout:(80/99): python3-kubernetes-26.1.0-3.el9.noarch 32 MB/s | 1.0 MB 00:00 2026-02-22T22:55:04.258 INFO:teuthology.orchestra.run.trial060.stdout:(81/99): python3-logutils-0.3.5-21.el9.noarch.r 1.4 MB/s | 46 kB 00:00 2026-02-22T22:55:04.275 INFO:teuthology.orchestra.run.trial060.stdout:(82/99): python3-more-itertools-8.12.0-2.el9.no 3.1 MB/s | 79 kB 00:00 2026-02-22T22:55:04.283 INFO:teuthology.orchestra.run.trial060.stdout:(83/99): python3-msgpack-1.0.3-2.el9.x86_64.rpm 3.4 MB/s | 86 kB 00:00 2026-02-22T22:55:04.283 INFO:teuthology.orchestra.run.trial060.stdout:(84/99): python3-natsort-7.1.1-5.el9.noarch.rpm 2.3 MB/s | 58 kB 00:00 2026-02-22T22:55:04.301 INFO:teuthology.orchestra.run.trial060.stdout:(85/99): python3-pecan-1.4.2-3.el9.noarch.rpm 10 MB/s | 272 kB 00:00 2026-02-22T22:55:04.307 INFO:teuthology.orchestra.run.trial060.stdout:(86/99): python3-portend-3.1.0-2.el9.noarch.rpm 675 kB/s | 16 kB 00:00 2026-02-22T22:55:04.308 INFO:teuthology.orchestra.run.trial060.stdout:(87/99): python3-pyOpenSSL-21.0.0-1.el9.noarch. 3.6 MB/s | 90 kB 00:00 2026-02-22T22:55:04.326 INFO:teuthology.orchestra.run.trial060.stdout:(88/99): python3-repoze-lru-0.7-16.el9.noarch.r 1.2 MB/s | 31 kB 00:00 2026-02-22T22:55:04.333 INFO:teuthology.orchestra.run.trial060.stdout:(89/99): python3-routes-2.5.1-5.el9.noarch.rpm 7.2 MB/s | 188 kB 00:00 2026-02-22T22:55:04.333 INFO:teuthology.orchestra.run.trial060.stdout:(90/99): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2026-02-22T22:55:04.351 INFO:teuthology.orchestra.run.trial060.stdout:(91/99): python3-saml-1.16.0-1.el9.noarch.rpm 4.9 MB/s | 125 kB 00:00 2026-02-22T22:55:04.358 INFO:teuthology.orchestra.run.trial060.stdout:(92/99): python3-tempora-5.0.0-2.el9.noarch.rpm 1.4 MB/s | 36 kB 00:00 2026-02-22T22:55:04.358 INFO:teuthology.orchestra.run.trial060.stdout:(93/99): python3-typing-extensions-4.15.0-1.el9 3.4 MB/s | 86 kB 00:00 2026-02-22T22:55:04.377 INFO:teuthology.orchestra.run.trial060.stdout:(94/99): python3-webob-1.8.8-2.el9.noarch.rpm 8.7 MB/s | 230 kB 00:00 2026-02-22T22:55:04.382 INFO:teuthology.orchestra.run.trial060.stdout:(95/99): python3-websocket-client-1.2.3-2.el9.n 3.5 MB/s | 90 kB 00:00 2026-02-22T22:55:04.385 INFO:teuthology.orchestra.run.trial060.stdout:(96/99): python3-werkzeug-2.0.3-3.el9.1.noarch. 15 MB/s | 427 kB 00:00 2026-02-22T22:55:04.402 INFO:teuthology.orchestra.run.trial060.stdout:(97/99): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 1.9 MB/s | 48 kB 00:00 2026-02-22T22:55:04.407 INFO:teuthology.orchestra.run.trial060.stdout:(98/99): python3-xmltodict-0.12.0-15.el9.noarch 909 kB/s | 22 kB 00:00 2026-02-22T22:55:04.411 INFO:teuthology.orchestra.run.trial060.stdout:(99/99): python3-zc-lockfile-2.0-10.el9.noarch. 779 kB/s | 20 kB 00:00 2026-02-22T22:55:04.415 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:04.415 INFO:teuthology.orchestra.run.trial060.stdout:Total 25 MB/s | 95 MB 00:03 2026-02-22T22:55:04.835 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:55:04.871 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:55:04.871 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:55:05.291 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:55:05.291 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:55:05.397 INFO:teuthology.orchestra.run.trial082.stdout:(36/99): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 764 kB/s | 3.0 MB 00:04 2026-02-22T22:55:05.771 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:55:05.774 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/99 2026-02-22T22:55:05.792 INFO:teuthology.orchestra.run.trial082.stdout:(37/99): python3-devel-3.9.25-3.el9.x86_64.rpm 618 kB/s | 244 kB 00:00 2026-02-22T22:55:05.805 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/99 2026-02-22T22:55:05.811 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 3/99 2026-02-22T22:55:05.832 INFO:teuthology.orchestra.run.trial060.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 4/99 2026-02-22T22:55:05.838 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 5/99 2026-02-22T22:55:05.871 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 6/99 2026-02-22T22:55:05.880 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 7/99 2026-02-22T22:55:05.885 INFO:teuthology.orchestra.run.trial060.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 8/99 2026-02-22T22:55:05.888 INFO:teuthology.orchestra.run.trial060.stdout: Installing : lua-5.4.4-4.el9.x86_64 9/99 2026-02-22T22:55:05.892 INFO:teuthology.orchestra.run.trial060.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 10/99 2026-02-22T22:55:05.915 INFO:teuthology.orchestra.run.trial060.stdout: Installing : unzip-6.0-59.el9.x86_64 11/99 2026-02-22T22:55:05.928 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 12/99 2026-02-22T22:55:05.931 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 13/99 2026-02-22T22:55:05.937 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 14/99 2026-02-22T22:55:05.939 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 15/99 2026-02-22T22:55:05.950 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 16/99 2026-02-22T22:55:05.956 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 17/99 2026-02-22T22:55:05.980 INFO:teuthology.orchestra.run.trial060.stdout: Installing : zip-3.0-35.el9.x86_64 18/99 2026-02-22T22:55:05.984 INFO:teuthology.orchestra.run.trial060.stdout: Installing : luarocks-3.9.2-5.el9.noarch 19/99 2026-02-22T22:55:05.991 INFO:teuthology.orchestra.run.trial060.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 20/99 2026-02-22T22:55:06.016 INFO:teuthology.orchestra.run.trial060.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 21/99 2026-02-22T22:55:06.066 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 22/99 2026-02-22T22:55:06.079 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 23/99 2026-02-22T22:55:06.081 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-rsa-4.9-2.el9.noarch 24/99 2026-02-22T22:55:06.086 INFO:teuthology.orchestra.run.trial060.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 25/99 2026-02-22T22:55:06.091 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 26/99 2026-02-22T22:55:06.096 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 27/99 2026-02-22T22:55:06.099 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 28/99 2026-02-22T22:55:06.113 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 29/99 2026-02-22T22:55:06.134 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 30/99 2026-02-22T22:55:06.139 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 31/99 2026-02-22T22:55:06.144 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 32/99 2026-02-22T22:55:06.155 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 33/99 2026-02-22T22:55:06.164 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 34/99 2026-02-22T22:55:06.170 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 35/99 2026-02-22T22:55:06.186 INFO:teuthology.orchestra.run.trial082.stdout:(38/99): python3-jinja2-2.11.3-8.el9.noarch.rpm 630 kB/s | 249 kB 00:00 2026-02-22T22:55:06.189 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 36/99 2026-02-22T22:55:06.199 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 37/99 2026-02-22T22:55:06.253 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 38/99 2026-02-22T22:55:06.265 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 39/99 2026-02-22T22:55:06.280 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 40/99 2026-02-22T22:55:06.285 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 41/99 2026-02-22T22:55:06.293 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 42/99 2026-02-22T22:55:06.333 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 43/99 2026-02-22T22:55:06.503 INFO:teuthology.orchestra.run.trial082.stdout:(39/99): python3-libstoragemgmt-1.10.1-1.el9.x8 559 kB/s | 177 kB 00:00 2026-02-22T22:55:06.656 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 44/99 2026-02-22T22:55:06.666 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 45/99 2026-02-22T22:55:06.670 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 46/99 2026-02-22T22:55:06.675 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 47/99 2026-02-22T22:55:06.677 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 48/99 2026-02-22T22:55:06.705 INFO:teuthology.orchestra.run.trial060.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 49/99 2026-02-22T22:55:06.749 INFO:teuthology.orchestra.run.trial060.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 50/99 2026-02-22T22:55:06.756 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 51/99 2026-02-22T22:55:06.759 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 52/99 2026-02-22T22:55:06.765 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 53/99 2026-02-22T22:55:06.768 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 54/99 2026-02-22T22:55:06.775 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 55/99 2026-02-22T22:55:06.778 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 56/99 2026-02-22T22:55:06.806 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 57/99 2026-02-22T22:55:06.818 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 58/99 2026-02-22T22:55:06.852 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 59/99 2026-02-22T22:55:07.075 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 60/99 2026-02-22T22:55:07.098 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 61/99 2026-02-22T22:55:07.102 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 62/99 2026-02-22T22:55:07.162 INFO:teuthology.orchestra.run.trial060.stdout: Installing : openblas-0.3.29-1.el9.x86_64 63/99 2026-02-22T22:55:07.164 INFO:teuthology.orchestra.run.trial060.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 64/99 2026-02-22T22:55:07.185 INFO:teuthology.orchestra.run.trial060.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 65/99 2026-02-22T22:55:07.513 INFO:teuthology.orchestra.run.trial060.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 66/99 2026-02-22T22:55:07.587 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 67/99 2026-02-22T22:55:07.995 INFO:teuthology.orchestra.run.trial082.stdout:(40/99): python3-lxml-4.6.5-3.el9.x86_64.rpm 839 kB/s | 1.2 MB 00:01 2026-02-22T22:55:08.264 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 68/99 2026-02-22T22:55:08.280 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 69/99 2026-02-22T22:55:08.291 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 70/99 2026-02-22T22:55:08.297 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-packaging-20.9-5.el9.noarch 71/99 2026-02-22T22:55:08.311 INFO:teuthology.orchestra.run.trial082.stdout:(41/99): python3-mako-1.1.4-6.el9.noarch.rpm 545 kB/s | 172 kB 00:00 2026-02-22T22:55:08.311 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-ply-3.11-14.el9.noarch 72/99 2026-02-22T22:55:08.327 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 73/99 2026-02-22T22:55:08.395 INFO:teuthology.orchestra.run.trial082.stdout:(42/99): openblas-openmp-0.3.29-1.el9.x86_64.rp 893 kB/s | 5.3 MB 00:06 2026-02-22T22:55:08.403 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 74/99 2026-02-22T22:55:08.411 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 75/99 2026-02-22T22:55:08.435 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/99 2026-02-22T22:55:08.464 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 77/99 2026-02-22T22:55:08.475 INFO:teuthology.orchestra.run.trial082.stdout:(43/99): python3-markupsafe-1.1.1-12.el9.x86_64 212 kB/s | 35 kB 00:00 2026-02-22T22:55:08.515 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 78/99 2026-02-22T22:55:08.521 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 79/99 2026-02-22T22:55:08.524 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 80/99 2026-02-22T22:55:08.526 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 81/99 2026-02-22T22:55:08.543 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:55:08.543 INFO:teuthology.orchestra.run.trial060.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-22T22:55:08.543 INFO:teuthology.orchestra.run.trial060.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-22T22:55:08.543 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:08.550 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:55:08.575 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:55:08.575 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-22T22:55:08.576 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:08.609 INFO:teuthology.orchestra.run.trial060.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 83/99 2026-02-22T22:55:08.673 INFO:teuthology.orchestra.run.trial060.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 84/99 2026-02-22T22:55:08.675 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:55:08.689 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:55:08.690 INFO:teuthology.orchestra.run.trial060.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:08.690 INFO:teuthology.orchestra.run.trial060.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-22T22:55:08.690 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:09.104 INFO:teuthology.orchestra.run.trial082.stdout:(44/99): python3-numpy-f2py-1.23.5-2.el9.x86_64 703 kB/s | 442 kB 00:00 2026-02-22T22:55:09.342 INFO:teuthology.orchestra.run.trial082.stdout:(45/99): python3-packaging-20.9-5.el9.noarch.rp 325 kB/s | 77 kB 00:00 2026-02-22T22:55:09.387 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:55:09.413 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:55:09.413 INFO:teuthology.orchestra.run.trial060.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:09.414 INFO:teuthology.orchestra.run.trial060.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-22T22:55:09.414 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:55:09.414 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:55:09.414 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:09.472 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:55:09.474 INFO:teuthology.orchestra.run.trial060.stdout: Installing : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:55:09.479 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 88/99 2026-02-22T22:55:09.499 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 89/99 2026-02-22T22:55:09.500 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:55:09.736 INFO:teuthology.orchestra.run.trial082.stdout:(46/99): python3-protobuf-3.14.0-17.el9.noarch. 679 kB/s | 267 kB 00:00 2026-02-22T22:55:09.996 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:55:09.999 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:55:10.051 INFO:teuthology.orchestra.run.trial082.stdout:(47/99): python3-pyasn1-0.4.8-6.el9.noarch.rpm 503 kB/s | 159 kB 00:00 2026-02-22T22:55:10.446 INFO:teuthology.orchestra.run.trial082.stdout:(48/99): python3-pyasn1-modules-0.4.8-6.el9.noa 707 kB/s | 279 kB 00:00 2026-02-22T22:55:10.456 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:55:10.457 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:55:10.471 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:55:10.471 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:55:10.527 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:55:10.576 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 94/99 2026-02-22T22:55:10.577 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:55:10.599 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:55:10.599 INFO:teuthology.orchestra.run.trial060.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:10.599 INFO:teuthology.orchestra.run.trial060.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-22T22:55:10.600 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:55:10.600 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:55:10.600 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:10.605 INFO:teuthology.orchestra.run.trial082.stdout:(49/99): python3-babel-2.9.1-2.el9.noarch.rpm 889 kB/s | 6.0 MB 00:06 2026-02-22T22:55:10.606 INFO:teuthology.orchestra.run.trial082.stdout:(50/99): python3-requests-oauthlib-1.3.0-12.el9 334 kB/s | 54 kB 00:00 2026-02-22T22:55:10.612 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:55:10.827 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:55:10.829 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:55:10.842 INFO:teuthology.orchestra.run.trial082.stdout:(51/99): python3-toml-0.10.2-6.el9.noarch.rpm 176 kB/s | 42 kB 00:00 2026-02-22T22:55:10.850 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:55:10.850 INFO:teuthology.orchestra.run.trial060.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:10.850 INFO:teuthology.orchestra.run.trial060.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-22T22:55:10.850 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:55:10.850 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:55:10.850 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:10.951 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:55:10.972 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:55:10.972 INFO:teuthology.orchestra.run.trial060.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:10.972 INFO:teuthology.orchestra.run.trial060.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-22T22:55:10.972 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:55:10.972 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:55:10.973 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:10.991 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:55:11.157 INFO:teuthology.orchestra.run.trial082.stdout:(52/99): xmlsec1-1.2.29-13.el9.x86_64.rpm 599 kB/s | 189 kB 00:00 2026-02-22T22:55:11.394 INFO:teuthology.orchestra.run.trial082.stdout:(53/99): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 382 kB/s | 90 kB 00:00 2026-02-22T22:55:11.464 INFO:teuthology.orchestra.run.trial082.stdout:(54/99): lua-devel-5.4.4-4.el9.x86_64.rpm 321 kB/s | 22 kB 00:00 2026-02-22T22:55:11.563 INFO:teuthology.orchestra.run.trial082.stdout:(55/99): protobuf-compiler-3.14.0-17.el9.x86_64 8.5 MB/s | 862 kB 00:00 2026-02-22T22:55:11.672 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:55:11.673 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 1/99 2026-02-22T22:55:11.673 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2/99 2026-02-22T22:55:11.673 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 3/99 2026-02-22T22:55:11.673 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 4/99 2026-02-22T22:55:11.673 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 5/99 2026-02-22T22:55:11.673 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 6/99 2026-02-22T22:55:11.673 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 7/99 2026-02-22T22:55:11.673 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 8/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 9/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 10/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 11/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 12/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 13/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 14/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 15/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/99 2026-02-22T22:55:11.674 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : zip-3.0-35.el9.x86_64 29/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/99 2026-02-22T22:55:11.675 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 39/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 41/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 42/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 43/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/99 2026-02-22T22:55:11.676 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/99 2026-02-22T22:55:11.677 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/99 2026-02-22T22:55:11.678 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 81/99 2026-02-22T22:55:11.679 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 82/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 83/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 84/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 85/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 86/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 88/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 89/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 90/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 91/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 92/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 93/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 94/99 2026-02-22T22:55:11.680 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 95/99 2026-02-22T22:55:11.681 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 96/99 2026-02-22T22:55:11.681 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 97/99 2026-02-22T22:55:11.681 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 98/99 2026-02-22T22:55:11.735 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 99/99 2026-02-22T22:55:11.735 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:11.735 INFO:teuthology.orchestra.run.trial060.stdout:Installed: 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:11.736 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-22T22:55:11.737 INFO:teuthology.orchestra.run.trial060.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: lua-5.4.4-4.el9.x86_64 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-22T22:55:11.738 INFO:teuthology.orchestra.run.trial060.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-22T22:55:11.739 INFO:teuthology.orchestra.run.trial060.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-22T22:55:11.740 INFO:teuthology.orchestra.run.trial060.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-logutils-0.3.5-21.el9.noarch 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-mako-1.1.4-6.el9.noarch 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-pecan-1.4.2-3.el9.noarch 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-ply-3.11-14.el9.noarch 2026-02-22T22:55:11.741 INFO:teuthology.orchestra.run.trial060.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-22T22:55:11.742 INFO:teuthology.orchestra.run.trial060.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-webob-1.8.8-2.el9.noarch 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-22T22:55:11.743 INFO:teuthology.orchestra.run.trial060.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-22T22:55:11.744 INFO:teuthology.orchestra.run.trial060.stdout: unzip-6.0-59.el9.x86_64 2026-02-22T22:55:11.744 INFO:teuthology.orchestra.run.trial060.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-22T22:55:11.744 INFO:teuthology.orchestra.run.trial060.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-22T22:55:11.744 INFO:teuthology.orchestra.run.trial060.stdout: zip-3.0-35.el9.x86_64 2026-02-22T22:55:11.744 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:11.744 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:11.791 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-base 2026-02-22T22:55:12.101 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:31 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:12.169 INFO:teuthology.orchestra.run.trial060.stdout:Package ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:12.180 INFO:teuthology.orchestra.run.trial082.stdout:(56/99): abseil-cpp-20211102.0-4.el9.x86_64.rpm 893 kB/s | 551 kB 00:00 2026-02-22T22:55:12.187 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:12.188 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:12.188 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:12.209 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install cephadm 2026-02-22T22:55:12.261 INFO:teuthology.orchestra.run.trial082.stdout:(57/99): grpc-data-1.46.7-10.el9.noarch.rpm 240 kB/s | 19 kB 00:00 2026-02-22T22:55:12.355 INFO:teuthology.orchestra.run.trial082.stdout:(58/99): luarocks-3.9.2-5.el9.noarch.rpm 1.6 MB/s | 151 kB 00:00 2026-02-22T22:55:12.491 INFO:teuthology.orchestra.run.trial082.stdout:(59/99): python3-asyncssh-2.13.2-5.el9.noarch.r 3.9 MB/s | 548 kB 00:00 2026-02-22T22:55:12.516 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:31 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:12.573 INFO:teuthology.orchestra.run.trial082.stdout:(60/99): python3-autocommand-2.2.2-8.el9.noarch 360 kB/s | 29 kB 00:00 2026-02-22T22:55:12.580 INFO:teuthology.orchestra.run.trial060.stdout:Package cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:12.598 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:12.599 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:12.599 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:12.620 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-immutable-object-cache 2026-02-22T22:55:12.657 INFO:teuthology.orchestra.run.trial082.stdout:(61/99): python3-backports-tarfile-1.2.0-1.el9. 711 kB/s | 60 kB 00:00 2026-02-22T22:55:12.740 INFO:teuthology.orchestra.run.trial082.stdout:(62/99): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 524 kB/s | 43 kB 00:00 2026-02-22T22:55:12.823 INFO:teuthology.orchestra.run.trial082.stdout:(63/99): python3-cachetools-4.2.4-1.el9.noarch. 392 kB/s | 32 kB 00:00 2026-02-22T22:55:12.904 INFO:teuthology.orchestra.run.trial082.stdout:(64/99): python3-certifi-2023.05.07-4.el9.noarc 174 kB/s | 14 kB 00:00 2026-02-22T22:55:12.924 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:31 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:12.998 INFO:teuthology.orchestra.run.trial082.stdout:(65/99): python3-cheroot-10.0.1-4.el9.noarch.rp 1.8 MB/s | 173 kB 00:00 2026-02-22T22:55:13.007 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:13.008 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:13.008 INFO:teuthology.orchestra.run.trial060.stdout: Package Arch Version Repo Size 2026-02-22T22:55:13.008 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:13.008 INFO:teuthology.orchestra.run.trial060.stdout:Installing: 2026-02-22T22:55:13.009 INFO:teuthology.orchestra.run.trial060.stdout: ceph-immutable-object-cache x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 145 k 2026-02-22T22:55:13.009 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:13.009 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:55:13.009 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:13.009 INFO:teuthology.orchestra.run.trial060.stdout:Install 1 Package 2026-02-22T22:55:13.009 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:13.009 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 145 k 2026-02-22T22:55:13.009 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 431 k 2026-02-22T22:55:13.009 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:55:13.109 INFO:teuthology.orchestra.run.trial060.stdout:ceph-immutable-object-cache-19.2.3-689.gf90f4ea 1.4 MB/s | 145 kB 00:00 2026-02-22T22:55:13.109 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:13.109 INFO:teuthology.orchestra.run.trial060.stdout:Total 1.4 MB/s | 145 kB 00:00 2026-02-22T22:55:13.109 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:55:13.110 INFO:teuthology.orchestra.run.trial082.stdout:(66/99): python3-cherrypy-18.6.1-2.el9.noarch.r 3.2 MB/s | 358 kB 00:00 2026-02-22T22:55:13.112 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:55:13.112 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:55:13.143 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:55:13.144 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:55:13.198 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:55:13.200 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:55:13.209 INFO:teuthology.orchestra.run.trial082.stdout:(67/99): python3-google-auth-2.45.0-1.el9.noarc 2.5 MB/s | 254 kB 00:00 2026-02-22T22:55:13.221 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:55:13.221 INFO:teuthology.orchestra.run.trial060.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:13.221 INFO:teuthology.orchestra.run.trial060.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-22T22:55:13.221 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:13.454 INFO:teuthology.orchestra.run.trial082.stdout:(68/99): python3-grpcio-1.46.7-10.el9.x86_64.rp 8.3 MB/s | 2.0 MB 00:00 2026-02-22T22:55:13.537 INFO:teuthology.orchestra.run.trial082.stdout:(69/99): python3-grpcio-tools-1.46.7-10.el9.x86 1.7 MB/s | 144 kB 00:00 2026-02-22T22:55:13.542 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:55:13.542 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:13.542 INFO:teuthology.orchestra.run.trial060.stdout:Installed: 2026-02-22T22:55:13.542 INFO:teuthology.orchestra.run.trial060.stdout: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:13.542 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:13.542 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:13.568 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-mgr 2026-02-22T22:55:13.620 INFO:teuthology.orchestra.run.trial082.stdout:(70/99): python3-influxdb-5.3.1-1.el9.noarch.rp 1.6 MB/s | 139 kB 00:00 2026-02-22T22:55:13.702 INFO:teuthology.orchestra.run.trial082.stdout:(71/99): python3-isodate-0.6.1-3.el9.noarch.rpm 689 kB/s | 56 kB 00:00 2026-02-22T22:55:13.782 INFO:teuthology.orchestra.run.trial082.stdout:(72/99): python3-jaraco-8.2.1-3.el9.noarch.rpm 133 kB/s | 11 kB 00:00 2026-02-22T22:55:13.864 INFO:teuthology.orchestra.run.trial082.stdout:(73/99): python3-jaraco-classes-3.2.1-5.el9.noa 218 kB/s | 18 kB 00:00 2026-02-22T22:55:13.880 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:32 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:13.944 INFO:teuthology.orchestra.run.trial060.stdout:Package ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:13.945 INFO:teuthology.orchestra.run.trial082.stdout:(74/99): python3-jaraco-collections-3.0.0-8.el9 286 kB/s | 23 kB 00:00 2026-02-22T22:55:13.962 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:13.963 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:13.963 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:13.983 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-mgr-dashboard 2026-02-22T22:55:14.025 INFO:teuthology.orchestra.run.trial082.stdout:(75/99): python3-jaraco-context-6.0.1-3.el9.noa 243 kB/s | 20 kB 00:00 2026-02-22T22:55:14.106 INFO:teuthology.orchestra.run.trial082.stdout:(76/99): python3-jaraco-functools-3.5.0-2.el9.n 241 kB/s | 19 kB 00:00 2026-02-22T22:55:14.187 INFO:teuthology.orchestra.run.trial082.stdout:(77/99): python3-jaraco-text-4.0.0-2.el9.noarch 327 kB/s | 26 kB 00:00 2026-02-22T22:55:14.293 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:33 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:14.293 INFO:teuthology.orchestra.run.trial082.stdout:(78/99): python3-kubernetes-26.1.0-3.el9.noarch 9.6 MB/s | 1.0 MB 00:00 2026-02-22T22:55:14.357 INFO:teuthology.orchestra.run.trial060.stdout:Package ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:14.375 INFO:teuthology.orchestra.run.trial082.stdout:(79/99): python3-logutils-0.3.5-21.el9.noarch.r 569 kB/s | 46 kB 00:00 2026-02-22T22:55:14.375 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:14.376 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:14.376 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:14.396 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-22T22:55:14.456 INFO:teuthology.orchestra.run.trial082.stdout:(80/99): python3-more-itertools-8.12.0-2.el9.no 966 kB/s | 79 kB 00:00 2026-02-22T22:55:14.538 INFO:teuthology.orchestra.run.trial082.stdout:(81/99): python3-msgpack-1.0.3-2.el9.x86_64.rpm 1.0 MB/s | 86 kB 00:00 2026-02-22T22:55:14.619 INFO:teuthology.orchestra.run.trial082.stdout:(82/99): python3-natsort-7.1.1-5.el9.noarch.rpm 711 kB/s | 58 kB 00:00 2026-02-22T22:55:14.706 INFO:teuthology.orchestra.run.trial082.stdout:(83/99): python3-pecan-1.4.2-3.el9.noarch.rpm 3.1 MB/s | 272 kB 00:00 2026-02-22T22:55:14.707 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:33 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:14.772 INFO:teuthology.orchestra.run.trial060.stdout:Package ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:14.786 INFO:teuthology.orchestra.run.trial082.stdout:(84/99): python3-portend-3.1.0-2.el9.noarch.rpm 203 kB/s | 16 kB 00:00 2026-02-22T22:55:14.790 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:14.791 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:14.791 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:14.812 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-mgr-rook 2026-02-22T22:55:14.869 INFO:teuthology.orchestra.run.trial082.stdout:(85/99): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.1 MB/s | 90 kB 00:00 2026-02-22T22:55:14.949 INFO:teuthology.orchestra.run.trial082.stdout:(86/99): python3-repoze-lru-0.7-16.el9.noarch.r 381 kB/s | 31 kB 00:00 2026-02-22T22:55:15.034 INFO:teuthology.orchestra.run.trial082.stdout:(87/99): python3-routes-2.5.1-5.el9.noarch.rpm 2.2 MB/s | 188 kB 00:00 2026-02-22T22:55:15.115 INFO:teuthology.orchestra.run.trial082.stdout:(88/99): python3-rsa-4.9-2.el9.noarch.rpm 727 kB/s | 59 kB 00:00 2026-02-22T22:55:15.120 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:15.185 INFO:teuthology.orchestra.run.trial060.stdout:Package ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:15.197 INFO:teuthology.orchestra.run.trial082.stdout:(89/99): python3-saml-1.16.0-1.el9.noarch.rpm 1.5 MB/s | 125 kB 00:00 2026-02-22T22:55:15.203 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:15.204 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:15.204 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:15.226 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-mgr-cephadm 2026-02-22T22:55:15.278 INFO:teuthology.orchestra.run.trial082.stdout:(90/99): python3-tempora-5.0.0-2.el9.noarch.rpm 442 kB/s | 36 kB 00:00 2026-02-22T22:55:15.360 INFO:teuthology.orchestra.run.trial082.stdout:(91/99): python3-typing-extensions-4.15.0-1.el9 1.0 MB/s | 86 kB 00:00 2026-02-22T22:55:15.446 INFO:teuthology.orchestra.run.trial082.stdout:(92/99): python3-webob-1.8.8-2.el9.noarch.rpm 2.6 MB/s | 230 kB 00:00 2026-02-22T22:55:15.454 INFO:teuthology.orchestra.run.trial082.stdout:(93/99): python3-numpy-1.23.5-2.el9.x86_64.rpm 889 kB/s | 6.1 MB 00:07 2026-02-22T22:55:15.528 INFO:teuthology.orchestra.run.trial082.stdout:(94/99): python3-websocket-client-1.2.3-2.el9.n 1.1 MB/s | 90 kB 00:00 2026-02-22T22:55:15.540 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:15.605 INFO:teuthology.orchestra.run.trial060.stdout:Package ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:15.609 INFO:teuthology.orchestra.run.trial082.stdout:(95/99): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 598 kB/s | 48 kB 00:00 2026-02-22T22:55:15.623 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:15.623 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:15.624 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:15.645 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-fuse 2026-02-22T22:55:15.689 INFO:teuthology.orchestra.run.trial082.stdout:(96/99): python3-xmltodict-0.12.0-15.el9.noarch 275 kB/s | 22 kB 00:00 2026-02-22T22:55:15.770 INFO:teuthology.orchestra.run.trial082.stdout:(97/99): python3-zc-lockfile-2.0-10.el9.noarch. 247 kB/s | 20 kB 00:00 2026-02-22T22:55:15.894 INFO:teuthology.orchestra.run.trial082.stdout:(98/99): python3-werkzeug-2.0.3-3.el9.1.noarch. 972 kB/s | 427 kB 00:00 2026-02-22T22:55:15.954 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:16.037 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:16.038 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:16.038 INFO:teuthology.orchestra.run.trial060.stdout: Package Architecture Version Repository Size 2026-02-22T22:55:16.038 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:16.039 INFO:teuthology.orchestra.run.trial060.stdout:Installing: 2026-02-22T22:55:16.039 INFO:teuthology.orchestra.run.trial060.stdout: ceph-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.2 M 2026-02-22T22:55:16.039 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:16.039 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:55:16.039 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:16.039 INFO:teuthology.orchestra.run.trial060.stdout:Install 1 Package 2026-02-22T22:55:16.039 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:16.039 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 1.2 M 2026-02-22T22:55:16.039 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 3.6 M 2026-02-22T22:55:16.039 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:55:16.161 INFO:teuthology.orchestra.run.trial060.stdout:ceph-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 9.4 MB/s | 1.2 MB 00:00 2026-02-22T22:55:16.161 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:16.161 INFO:teuthology.orchestra.run.trial060.stdout:Total 9.4 MB/s | 1.2 MB 00:00 2026-02-22T22:55:16.161 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:55:16.165 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:55:16.165 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:55:16.201 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:55:16.201 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:55:16.303 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:55:16.321 INFO:teuthology.orchestra.run.trial060.stdout: Installing : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:16.558 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:16.614 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:16.614 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:16.615 INFO:teuthology.orchestra.run.trial060.stdout:Installed: 2026-02-22T22:55:16.615 INFO:teuthology.orchestra.run.trial060.stdout: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:16.615 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:16.615 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:16.643 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install ceph-volume 2026-02-22T22:55:16.954 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:35 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:17.019 INFO:teuthology.orchestra.run.trial060.stdout:Package ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:17.037 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:17.037 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:17.038 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:17.059 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install librados-devel 2026-02-22T22:55:17.369 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:17.452 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:17.453 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:17.453 INFO:teuthology.orchestra.run.trial060.stdout: Package Arch Version Repo Size 2026-02-22T22:55:17.453 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:17.453 INFO:teuthology.orchestra.run.trial060.stdout:Installing: 2026-02-22T22:55:17.453 INFO:teuthology.orchestra.run.trial060.stdout: librados-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 126 k 2026-02-22T22:55:17.453 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:17.453 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:55:17.453 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:17.453 INFO:teuthology.orchestra.run.trial060.stdout:Install 1 Package 2026-02-22T22:55:17.454 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:17.454 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 126 k 2026-02-22T22:55:17.454 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 456 k 2026-02-22T22:55:17.454 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:55:17.540 INFO:teuthology.orchestra.run.trial060.stdout:librados-devel-19.2.3-689.gf90f4eae.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-22T22:55:17.540 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:17.540 INFO:teuthology.orchestra.run.trial060.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-22T22:55:17.540 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:55:17.543 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:55:17.543 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:55:17.575 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:55:17.575 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:55:17.630 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:55:17.646 INFO:teuthology.orchestra.run.trial060.stdout: Installing : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:17.727 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:17.785 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:17.785 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:17.785 INFO:teuthology.orchestra.run.trial060.stdout:Installed: 2026-02-22T22:55:17.785 INFO:teuthology.orchestra.run.trial060.stdout: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:17.785 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:17.786 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:17.813 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install libcephfs2 2026-02-22T22:55:18.127 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:37 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:18.192 INFO:teuthology.orchestra.run.trial060.stdout:Package libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:18.210 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:18.211 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:18.211 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:18.232 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install libcephfs-devel 2026-02-22T22:55:18.543 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:37 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:18.626 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:18.627 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:18.627 INFO:teuthology.orchestra.run.trial060.stdout: Package Arch Version Repo Size 2026-02-22T22:55:18.627 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:18.627 INFO:teuthology.orchestra.run.trial060.stdout:Installing: 2026-02-22T22:55:18.628 INFO:teuthology.orchestra.run.trial060.stdout: libcephfs-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 34 k 2026-02-22T22:55:18.628 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:18.628 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:55:18.628 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:18.628 INFO:teuthology.orchestra.run.trial060.stdout:Install 1 Package 2026-02-22T22:55:18.628 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:18.628 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 34 k 2026-02-22T22:55:18.628 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 153 k 2026-02-22T22:55:18.628 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:55:18.693 INFO:teuthology.orchestra.run.trial060.stdout:libcephfs-devel-19.2.3-689.gf90f4eae.el9.x86_64 514 kB/s | 34 kB 00:00 2026-02-22T22:55:18.693 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:18.693 INFO:teuthology.orchestra.run.trial060.stdout:Total 508 kB/s | 34 kB 00:00 2026-02-22T22:55:18.693 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:55:18.695 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:55:18.695 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:55:18.698 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:55:18.698 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:55:18.719 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:55:18.734 INFO:teuthology.orchestra.run.trial060.stdout: Installing : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:18.788 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:18.845 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:18.845 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:18.845 INFO:teuthology.orchestra.run.trial060.stdout:Installed: 2026-02-22T22:55:18.845 INFO:teuthology.orchestra.run.trial060.stdout: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:18.845 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:18.845 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:18.872 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install librados2 2026-02-22T22:55:19.177 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:19.242 INFO:teuthology.orchestra.run.trial060.stdout:Package librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:19.260 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:19.261 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:19.261 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:19.282 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install librbd1 2026-02-22T22:55:19.590 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:19.654 INFO:teuthology.orchestra.run.trial060.stdout:Package librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:19.673 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:19.673 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:19.673 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:19.695 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install python3-rados 2026-02-22T22:55:20.000 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:20.064 INFO:teuthology.orchestra.run.trial060.stdout:Package python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:20.082 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:20.083 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:20.083 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:20.104 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install python3-rgw 2026-02-22T22:55:20.413 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:20.478 INFO:teuthology.orchestra.run.trial060.stdout:Package python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:20.496 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:20.497 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:20.497 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:20.518 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install python3-cephfs 2026-02-22T22:55:20.828 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:20.893 INFO:teuthology.orchestra.run.trial060.stdout:Package python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:20.911 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:20.912 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:20.912 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:20.918 INFO:teuthology.orchestra.run.trial082.stdout:(99/99): python3-scipy-1.9.3-2.el9.x86_64.rpm 1.9 MB/s | 19 MB 00:10 2026-02-22T22:55:20.921 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:20.921 INFO:teuthology.orchestra.run.trial082.stdout:Total 4.5 MB/s | 95 MB 00:21 2026-02-22T22:55:20.932 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install python3-rbd 2026-02-22T22:55:21.238 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:21.304 INFO:teuthology.orchestra.run.trial060.stdout:Package python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:21.322 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:21.323 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:21.323 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:21.340 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:55:21.344 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install rbd-fuse 2026-02-22T22:55:21.377 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:55:21.378 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:55:21.655 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:21.739 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:21.739 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:21.739 INFO:teuthology.orchestra.run.trial060.stdout: Package Architecture Version Repository Size 2026-02-22T22:55:21.739 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:21.739 INFO:teuthology.orchestra.run.trial060.stdout:Installing: 2026-02-22T22:55:21.739 INFO:teuthology.orchestra.run.trial060.stdout: rbd-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 85 k 2026-02-22T22:55:21.739 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:21.740 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:55:21.740 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:21.740 INFO:teuthology.orchestra.run.trial060.stdout:Install 1 Package 2026-02-22T22:55:21.740 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:21.740 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 85 k 2026-02-22T22:55:21.740 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 227 k 2026-02-22T22:55:21.740 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:55:21.791 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:55:21.792 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:55:21.815 INFO:teuthology.orchestra.run.trial060.stdout:rbd-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.1 MB/s | 85 kB 00:00 2026-02-22T22:55:21.815 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:21.815 INFO:teuthology.orchestra.run.trial060.stdout:Total 1.1 MB/s | 85 kB 00:00 2026-02-22T22:55:21.815 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:55:21.818 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:55:21.818 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:55:21.849 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:55:21.849 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:55:21.900 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:55:21.917 INFO:teuthology.orchestra.run.trial060.stdout: Installing : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:21.997 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:22.055 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:22.055 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:22.055 INFO:teuthology.orchestra.run.trial060.stdout:Installed: 2026-02-22T22:55:22.056 INFO:teuthology.orchestra.run.trial060.stdout: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:22.056 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:22.056 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:22.083 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install rbd-mirror 2026-02-22T22:55:22.253 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:55:22.255 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/99 2026-02-22T22:55:22.286 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/99 2026-02-22T22:55:22.291 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 3/99 2026-02-22T22:55:22.312 INFO:teuthology.orchestra.run.trial082.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 4/99 2026-02-22T22:55:22.318 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 5/99 2026-02-22T22:55:22.352 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 6/99 2026-02-22T22:55:22.362 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 7/99 2026-02-22T22:55:22.368 INFO:teuthology.orchestra.run.trial082.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 8/99 2026-02-22T22:55:22.370 INFO:teuthology.orchestra.run.trial082.stdout: Installing : lua-5.4.4-4.el9.x86_64 9/99 2026-02-22T22:55:22.374 INFO:teuthology.orchestra.run.trial082.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 10/99 2026-02-22T22:55:22.394 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:22.397 INFO:teuthology.orchestra.run.trial082.stdout: Installing : unzip-6.0-59.el9.x86_64 11/99 2026-02-22T22:55:22.410 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 12/99 2026-02-22T22:55:22.414 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 13/99 2026-02-22T22:55:22.422 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 14/99 2026-02-22T22:55:22.424 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 15/99 2026-02-22T22:55:22.435 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 16/99 2026-02-22T22:55:22.441 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 17/99 2026-02-22T22:55:22.465 INFO:teuthology.orchestra.run.trial082.stdout: Installing : zip-3.0-35.el9.x86_64 18/99 2026-02-22T22:55:22.468 INFO:teuthology.orchestra.run.trial082.stdout: Installing : luarocks-3.9.2-5.el9.noarch 19/99 2026-02-22T22:55:22.475 INFO:teuthology.orchestra.run.trial082.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 20/99 2026-02-22T22:55:22.477 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:22.477 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:22.477 INFO:teuthology.orchestra.run.trial060.stdout: Package Arch Version Repo Size 2026-02-22T22:55:22.477 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:22.477 INFO:teuthology.orchestra.run.trial060.stdout:Installing: 2026-02-22T22:55:22.477 INFO:teuthology.orchestra.run.trial060.stdout: rbd-mirror x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.1 M 2026-02-22T22:55:22.477 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:22.478 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:55:22.478 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:22.478 INFO:teuthology.orchestra.run.trial060.stdout:Install 1 Package 2026-02-22T22:55:22.478 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:22.478 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 3.1 M 2026-02-22T22:55:22.478 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 13 M 2026-02-22T22:55:22.478 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:55:22.500 INFO:teuthology.orchestra.run.trial082.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 21/99 2026-02-22T22:55:22.551 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 22/99 2026-02-22T22:55:22.563 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 23/99 2026-02-22T22:55:22.566 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-rsa-4.9-2.el9.noarch 24/99 2026-02-22T22:55:22.571 INFO:teuthology.orchestra.run.trial082.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 25/99 2026-02-22T22:55:22.576 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 26/99 2026-02-22T22:55:22.581 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 27/99 2026-02-22T22:55:22.583 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 28/99 2026-02-22T22:55:22.598 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 29/99 2026-02-22T22:55:22.619 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 30/99 2026-02-22T22:55:22.623 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 31/99 2026-02-22T22:55:22.628 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 32/99 2026-02-22T22:55:22.630 INFO:teuthology.orchestra.run.trial060.stdout:rbd-mirror-19.2.3-689.gf90f4eae.el9.x86_64.rpm 21 MB/s | 3.1 MB 00:00 2026-02-22T22:55:22.630 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:22.630 INFO:teuthology.orchestra.run.trial060.stdout:Total 20 MB/s | 3.1 MB 00:00 2026-02-22T22:55:22.630 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:55:22.634 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:55:22.634 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:55:22.639 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 33/99 2026-02-22T22:55:22.648 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 34/99 2026-02-22T22:55:22.654 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 35/99 2026-02-22T22:55:22.674 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 36/99 2026-02-22T22:55:22.676 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:55:22.677 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:55:22.683 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 37/99 2026-02-22T22:55:22.737 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 38/99 2026-02-22T22:55:22.749 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 39/99 2026-02-22T22:55:22.764 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 40/99 2026-02-22T22:55:22.769 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 41/99 2026-02-22T22:55:22.776 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 42/99 2026-02-22T22:55:22.817 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 43/99 2026-02-22T22:55:22.866 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:55:22.868 INFO:teuthology.orchestra.run.trial060.stdout: Installing : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:22.891 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:22.892 INFO:teuthology.orchestra.run.trial060.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:22.892 INFO:teuthology.orchestra.run.trial060.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-22T22:55:22.892 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:55:22.892 INFO:teuthology.orchestra.run.trial060.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:55:22.892 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:23.143 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 44/99 2026-02-22T22:55:23.153 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 45/99 2026-02-22T22:55:23.157 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 46/99 2026-02-22T22:55:23.163 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 47/99 2026-02-22T22:55:23.165 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 48/99 2026-02-22T22:55:23.193 INFO:teuthology.orchestra.run.trial082.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 49/99 2026-02-22T22:55:23.222 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:23.222 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:23.222 INFO:teuthology.orchestra.run.trial060.stdout:Installed: 2026-02-22T22:55:23.222 INFO:teuthology.orchestra.run.trial060.stdout: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:23.222 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:23.222 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:23.238 INFO:teuthology.orchestra.run.trial082.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 50/99 2026-02-22T22:55:23.245 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 51/99 2026-02-22T22:55:23.249 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 52/99 2026-02-22T22:55:23.252 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install rbd-nbd 2026-02-22T22:55:23.255 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 53/99 2026-02-22T22:55:23.258 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 54/99 2026-02-22T22:55:23.265 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 55/99 2026-02-22T22:55:23.268 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 56/99 2026-02-22T22:55:23.297 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 57/99 2026-02-22T22:55:23.306 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 58/99 2026-02-22T22:55:23.341 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 59/99 2026-02-22T22:55:23.568 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 60/99 2026-02-22T22:55:23.568 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:23.591 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 61/99 2026-02-22T22:55:23.595 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 62/99 2026-02-22T22:55:23.652 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:23.652 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:23.652 INFO:teuthology.orchestra.run.trial060.stdout: Package Architecture Version Repository Size 2026-02-22T22:55:23.652 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:23.652 INFO:teuthology.orchestra.run.trial060.stdout:Installing: 2026-02-22T22:55:23.652 INFO:teuthology.orchestra.run.trial060.stdout: rbd-nbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 170 k 2026-02-22T22:55:23.653 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:23.653 INFO:teuthology.orchestra.run.trial060.stdout:Transaction Summary 2026-02-22T22:55:23.653 INFO:teuthology.orchestra.run.trial060.stdout:================================================================================ 2026-02-22T22:55:23.653 INFO:teuthology.orchestra.run.trial060.stdout:Install 1 Package 2026-02-22T22:55:23.653 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:23.653 INFO:teuthology.orchestra.run.trial060.stdout:Total download size: 170 k 2026-02-22T22:55:23.653 INFO:teuthology.orchestra.run.trial060.stdout:Installed size: 490 k 2026-02-22T22:55:23.653 INFO:teuthology.orchestra.run.trial060.stdout:Downloading Packages: 2026-02-22T22:55:23.654 INFO:teuthology.orchestra.run.trial082.stdout: Installing : openblas-0.3.29-1.el9.x86_64 63/99 2026-02-22T22:55:23.655 INFO:teuthology.orchestra.run.trial082.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 64/99 2026-02-22T22:55:23.677 INFO:teuthology.orchestra.run.trial082.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 65/99 2026-02-22T22:55:23.739 INFO:teuthology.orchestra.run.trial060.stdout:rbd-nbd-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.9 MB/s | 170 kB 00:00 2026-02-22T22:55:23.739 INFO:teuthology.orchestra.run.trial060.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:23.739 INFO:teuthology.orchestra.run.trial060.stdout:Total 1.9 MB/s | 170 kB 00:00 2026-02-22T22:55:23.739 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction check 2026-02-22T22:55:23.743 INFO:teuthology.orchestra.run.trial060.stdout:Transaction check succeeded. 2026-02-22T22:55:23.743 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction test 2026-02-22T22:55:23.774 INFO:teuthology.orchestra.run.trial060.stdout:Transaction test succeeded. 2026-02-22T22:55:23.774 INFO:teuthology.orchestra.run.trial060.stdout:Running transaction 2026-02-22T22:55:23.829 INFO:teuthology.orchestra.run.trial060.stdout: Preparing : 1/1 2026-02-22T22:55:23.846 INFO:teuthology.orchestra.run.trial060.stdout: Installing : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:23.928 INFO:teuthology.orchestra.run.trial060.stdout: Running scriptlet: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:23.989 INFO:teuthology.orchestra.run.trial060.stdout: Verifying : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:23.989 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:23.989 INFO:teuthology.orchestra.run.trial060.stdout:Installed: 2026-02-22T22:55:23.990 INFO:teuthology.orchestra.run.trial060.stdout: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:23.990 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:23.990 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:24.004 INFO:teuthology.orchestra.run.trial082.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 66/99 2026-02-22T22:55:24.016 DEBUG:teuthology.orchestra.run.trial060:> sudo yum -y install rbd-mirror 2026-02-22T22:55:24.078 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 67/99 2026-02-22T22:55:24.327 INFO:teuthology.orchestra.run.trial060.stdout:Last metadata expiration check: 0:00:43 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:24.392 INFO:teuthology.orchestra.run.trial060.stdout:Package rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:24.410 INFO:teuthology.orchestra.run.trial060.stdout:Dependencies resolved. 2026-02-22T22:55:24.411 INFO:teuthology.orchestra.run.trial060.stdout:Nothing to do. 2026-02-22T22:55:24.411 INFO:teuthology.orchestra.run.trial060.stdout:Complete! 2026-02-22T22:55:24.436 DEBUG:teuthology.parallel:result is None 2026-02-22T22:55:24.754 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 68/99 2026-02-22T22:55:24.771 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 69/99 2026-02-22T22:55:24.781 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 70/99 2026-02-22T22:55:24.788 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-packaging-20.9-5.el9.noarch 71/99 2026-02-22T22:55:24.802 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-ply-3.11-14.el9.noarch 72/99 2026-02-22T22:55:24.817 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 73/99 2026-02-22T22:55:24.895 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 74/99 2026-02-22T22:55:24.903 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 75/99 2026-02-22T22:55:24.927 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/99 2026-02-22T22:55:24.957 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 77/99 2026-02-22T22:55:25.008 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 78/99 2026-02-22T22:55:25.015 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 79/99 2026-02-22T22:55:25.018 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 80/99 2026-02-22T22:55:25.019 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 81/99 2026-02-22T22:55:25.038 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:55:25.038 INFO:teuthology.orchestra.run.trial082.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-22T22:55:25.038 INFO:teuthology.orchestra.run.trial082.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-22T22:55:25.038 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:25.047 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:55:25.073 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:55:25.073 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-22T22:55:25.074 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:25.111 INFO:teuthology.orchestra.run.trial082.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 83/99 2026-02-22T22:55:25.175 INFO:teuthology.orchestra.run.trial082.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 84/99 2026-02-22T22:55:25.177 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:55:25.192 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:55:25.192 INFO:teuthology.orchestra.run.trial082.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:25.192 INFO:teuthology.orchestra.run.trial082.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-22T22:55:25.192 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:25.887 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:55:25.912 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:55:25.912 INFO:teuthology.orchestra.run.trial082.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:25.912 INFO:teuthology.orchestra.run.trial082.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-22T22:55:25.912 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:55:25.912 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:55:25.912 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:25.970 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:55:25.972 INFO:teuthology.orchestra.run.trial082.stdout: Installing : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:55:25.977 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 88/99 2026-02-22T22:55:25.996 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 89/99 2026-02-22T22:55:25.998 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:55:26.490 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:55:26.493 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:55:26.943 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:55:26.943 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:55:26.956 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:55:26.957 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:55:27.012 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:55:27.061 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 94/99 2026-02-22T22:55:27.062 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:55:27.083 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:55:27.083 INFO:teuthology.orchestra.run.trial082.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:27.083 INFO:teuthology.orchestra.run.trial082.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-22T22:55:27.083 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:55:27.083 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:55:27.083 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:27.094 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:55:27.312 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:55:27.313 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:55:27.334 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:55:27.334 INFO:teuthology.orchestra.run.trial082.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:27.335 INFO:teuthology.orchestra.run.trial082.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-22T22:55:27.335 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:55:27.335 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:55:27.335 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:27.435 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:55:27.455 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:55:27.455 INFO:teuthology.orchestra.run.trial082.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:27.455 INFO:teuthology.orchestra.run.trial082.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-22T22:55:27.455 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:55:27.455 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:55:27.455 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:27.473 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:55:28.132 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:55:28.132 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 1/99 2026-02-22T22:55:28.132 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2/99 2026-02-22T22:55:28.132 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 3/99 2026-02-22T22:55:28.132 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 4/99 2026-02-22T22:55:28.132 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 5/99 2026-02-22T22:55:28.132 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 6/99 2026-02-22T22:55:28.132 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 7/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 8/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 9/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 10/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 11/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 12/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 13/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 14/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 15/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/99 2026-02-22T22:55:28.133 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : zip-3.0-35.el9.x86_64 29/99 2026-02-22T22:55:28.134 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 39/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 41/99 2026-02-22T22:55:28.135 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 42/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 43/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/99 2026-02-22T22:55:28.136 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/99 2026-02-22T22:55:28.137 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/99 2026-02-22T22:55:28.138 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 81/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 82/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 83/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 84/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 85/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 86/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 88/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 89/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 90/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 91/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 92/99 2026-02-22T22:55:28.139 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 93/99 2026-02-22T22:55:28.140 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 94/99 2026-02-22T22:55:28.140 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 95/99 2026-02-22T22:55:28.140 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 96/99 2026-02-22T22:55:28.140 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 97/99 2026-02-22T22:55:28.140 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 98/99 2026-02-22T22:55:28.195 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 99/99 2026-02-22T22:55:28.195 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:28.195 INFO:teuthology.orchestra.run.trial082.stdout:Installed: 2026-02-22T22:55:28.196 INFO:teuthology.orchestra.run.trial082.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-22T22:55:28.196 INFO:teuthology.orchestra.run.trial082.stdout: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:28.196 INFO:teuthology.orchestra.run.trial082.stdout: ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:28.196 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:28.196 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:28.196 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:28.196 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:28.196 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:28.196 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:28.196 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-22T22:55:28.197 INFO:teuthology.orchestra.run.trial082.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: lua-5.4.4-4.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-22T22:55:28.198 INFO:teuthology.orchestra.run.trial082.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-22T22:55:28.199 INFO:teuthology.orchestra.run.trial082.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-22T22:55:28.200 INFO:teuthology.orchestra.run.trial082.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-logutils-0.3.5-21.el9.noarch 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-mako-1.1.4-6.el9.noarch 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-pecan-1.4.2-3.el9.noarch 2026-02-22T22:55:28.201 INFO:teuthology.orchestra.run.trial082.stdout: python3-ply-3.11-14.el9.noarch 2026-02-22T22:55:28.202 INFO:teuthology.orchestra.run.trial082.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-22T22:55:28.202 INFO:teuthology.orchestra.run.trial082.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-22T22:55:28.202 INFO:teuthology.orchestra.run.trial082.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-22T22:55:28.202 INFO:teuthology.orchestra.run.trial082.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-22T22:55:28.202 INFO:teuthology.orchestra.run.trial082.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-22T22:55:28.202 INFO:teuthology.orchestra.run.trial082.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-22T22:55:28.202 INFO:teuthology.orchestra.run.trial082.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-22T22:55:28.202 INFO:teuthology.orchestra.run.trial082.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-22T22:55:28.202 INFO:teuthology.orchestra.run.trial082.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-webob-1.8.8-2.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-22T22:55:28.203 INFO:teuthology.orchestra.run.trial082.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2026-02-22T22:55:28.204 INFO:teuthology.orchestra.run.trial082.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-22T22:55:28.204 INFO:teuthology.orchestra.run.trial082.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-22T22:55:28.204 INFO:teuthology.orchestra.run.trial082.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-22T22:55:28.204 INFO:teuthology.orchestra.run.trial082.stdout: unzip-6.0-59.el9.x86_64 2026-02-22T22:55:28.204 INFO:teuthology.orchestra.run.trial082.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-22T22:55:28.204 INFO:teuthology.orchestra.run.trial082.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-22T22:55:28.204 INFO:teuthology.orchestra.run.trial082.stdout: zip-3.0-35.el9.x86_64 2026-02-22T22:55:28.204 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:28.204 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:28.252 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-base 2026-02-22T22:55:28.562 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:47 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:28.627 INFO:teuthology.orchestra.run.trial082.stdout:Package ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:28.646 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:28.650 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:28.650 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:28.667 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install cephadm 2026-02-22T22:55:28.978 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:47 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:29.043 INFO:teuthology.orchestra.run.trial082.stdout:Package cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:29.062 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:29.062 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:29.062 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:29.083 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-immutable-object-cache 2026-02-22T22:55:29.393 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:48 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:29.476 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:29.476 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:29.476 INFO:teuthology.orchestra.run.trial082.stdout: Package Arch Version Repo Size 2026-02-22T22:55:29.476 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:29.477 INFO:teuthology.orchestra.run.trial082.stdout:Installing: 2026-02-22T22:55:29.477 INFO:teuthology.orchestra.run.trial082.stdout: ceph-immutable-object-cache x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 145 k 2026-02-22T22:55:29.477 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:29.477 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:55:29.477 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:29.477 INFO:teuthology.orchestra.run.trial082.stdout:Install 1 Package 2026-02-22T22:55:29.477 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:29.477 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 145 k 2026-02-22T22:55:29.477 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 431 k 2026-02-22T22:55:29.477 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:55:29.565 INFO:teuthology.orchestra.run.trial082.stdout:ceph-immutable-object-cache-19.2.3-689.gf90f4ea 1.6 MB/s | 145 kB 00:00 2026-02-22T22:55:29.565 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:29.566 INFO:teuthology.orchestra.run.trial082.stdout:Total 1.6 MB/s | 145 kB 00:00 2026-02-22T22:55:29.566 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:55:29.569 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:55:29.569 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:55:29.600 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:55:29.601 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:55:29.655 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:55:29.657 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:55:29.678 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:55:29.678 INFO:teuthology.orchestra.run.trial082.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:29.678 INFO:teuthology.orchestra.run.trial082.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-22T22:55:29.678 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:29.967 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:55:29.968 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:29.968 INFO:teuthology.orchestra.run.trial082.stdout:Installed: 2026-02-22T22:55:29.968 INFO:teuthology.orchestra.run.trial082.stdout: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:29.968 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:29.968 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:29.995 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-mgr 2026-02-22T22:55:30.310 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:49 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:30.376 INFO:teuthology.orchestra.run.trial082.stdout:Package ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:30.394 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:30.395 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:30.395 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:30.416 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-mgr-dashboard 2026-02-22T22:55:30.728 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:49 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:30.795 INFO:teuthology.orchestra.run.trial082.stdout:Package ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:30.813 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:30.814 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:30.814 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:30.835 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-22T22:55:31.144 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:50 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:31.208 INFO:teuthology.orchestra.run.trial082.stdout:Package ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:31.226 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:31.227 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:31.227 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:31.248 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-mgr-rook 2026-02-22T22:55:31.557 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:50 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:31.622 INFO:teuthology.orchestra.run.trial082.stdout:Package ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:31.641 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:31.641 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:31.641 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:31.663 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-mgr-cephadm 2026-02-22T22:55:31.970 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:50 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:32.036 INFO:teuthology.orchestra.run.trial082.stdout:Package ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:32.054 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:32.055 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:32.055 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:32.076 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-fuse 2026-02-22T22:55:32.388 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:51 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:32.473 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:32.473 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:32.473 INFO:teuthology.orchestra.run.trial082.stdout: Package Architecture Version Repository Size 2026-02-22T22:55:32.473 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:32.473 INFO:teuthology.orchestra.run.trial082.stdout:Installing: 2026-02-22T22:55:32.473 INFO:teuthology.orchestra.run.trial082.stdout: ceph-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.2 M 2026-02-22T22:55:32.473 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:32.474 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:55:32.474 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:32.474 INFO:teuthology.orchestra.run.trial082.stdout:Install 1 Package 2026-02-22T22:55:32.474 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:32.474 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 1.2 M 2026-02-22T22:55:32.474 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 3.6 M 2026-02-22T22:55:32.474 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:55:32.824 INFO:teuthology.orchestra.run.trial082.stdout:ceph-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 3.3 MB/s | 1.2 MB 00:00 2026-02-22T22:55:32.824 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:32.824 INFO:teuthology.orchestra.run.trial082.stdout:Total 3.3 MB/s | 1.2 MB 00:00 2026-02-22T22:55:32.824 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:55:32.827 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:55:32.827 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:55:32.864 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:55:32.864 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:55:32.967 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:55:32.984 INFO:teuthology.orchestra.run.trial082.stdout: Installing : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:33.241 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:33.297 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:33.298 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:33.298 INFO:teuthology.orchestra.run.trial082.stdout:Installed: 2026-02-22T22:55:33.298 INFO:teuthology.orchestra.run.trial082.stdout: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:33.298 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:33.298 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:33.327 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install ceph-volume 2026-02-22T22:55:33.641 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:52 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:33.706 INFO:teuthology.orchestra.run.trial082.stdout:Package ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:55:33.725 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:33.725 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:33.725 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:33.746 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install librados-devel 2026-02-22T22:55:34.052 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:53 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:34.136 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:34.136 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:34.136 INFO:teuthology.orchestra.run.trial082.stdout: Package Arch Version Repo Size 2026-02-22T22:55:34.136 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:34.137 INFO:teuthology.orchestra.run.trial082.stdout:Installing: 2026-02-22T22:55:34.137 INFO:teuthology.orchestra.run.trial082.stdout: librados-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 126 k 2026-02-22T22:55:34.137 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:34.137 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:55:34.137 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:34.137 INFO:teuthology.orchestra.run.trial082.stdout:Install 1 Package 2026-02-22T22:55:34.137 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:34.137 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 126 k 2026-02-22T22:55:34.137 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 456 k 2026-02-22T22:55:34.137 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:55:34.494 INFO:teuthology.orchestra.run.trial082.stdout:librados-devel-19.2.3-689.gf90f4eae.el9.x86_64. 354 kB/s | 126 kB 00:00 2026-02-22T22:55:34.494 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:34.494 INFO:teuthology.orchestra.run.trial082.stdout:Total 353 kB/s | 126 kB 00:00 2026-02-22T22:55:34.495 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:55:34.498 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:55:34.498 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:55:34.530 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:55:34.530 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:55:34.585 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:55:34.601 INFO:teuthology.orchestra.run.trial082.stdout: Installing : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:34.683 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:34.741 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:34.741 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:34.742 INFO:teuthology.orchestra.run.trial082.stdout:Installed: 2026-02-22T22:55:34.742 INFO:teuthology.orchestra.run.trial082.stdout: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:34.742 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:34.742 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:34.769 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install libcephfs2 2026-02-22T22:55:35.094 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:54 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:35.160 INFO:teuthology.orchestra.run.trial082.stdout:Package libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:35.179 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:35.179 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:35.180 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:35.202 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install libcephfs-devel 2026-02-22T22:55:35.510 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:54 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:35.595 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:35.595 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:35.595 INFO:teuthology.orchestra.run.trial082.stdout: Package Arch Version Repo Size 2026-02-22T22:55:35.595 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:35.596 INFO:teuthology.orchestra.run.trial082.stdout:Installing: 2026-02-22T22:55:35.596 INFO:teuthology.orchestra.run.trial082.stdout: libcephfs-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 34 k 2026-02-22T22:55:35.596 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:35.596 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:55:35.596 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:35.596 INFO:teuthology.orchestra.run.trial082.stdout:Install 1 Package 2026-02-22T22:55:35.596 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:35.596 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 34 k 2026-02-22T22:55:35.596 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 153 k 2026-02-22T22:55:35.596 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:55:35.881 INFO:teuthology.orchestra.run.trial082.stdout:libcephfs-devel-19.2.3-689.gf90f4eae.el9.x86_64 117 kB/s | 34 kB 00:00 2026-02-22T22:55:35.882 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:35.882 INFO:teuthology.orchestra.run.trial082.stdout:Total 117 kB/s | 34 kB 00:00 2026-02-22T22:55:35.882 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:55:35.884 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:55:35.884 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:55:35.888 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:55:35.888 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:55:35.909 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:55:35.924 INFO:teuthology.orchestra.run.trial082.stdout: Installing : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:35.981 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:36.037 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:36.038 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:36.038 INFO:teuthology.orchestra.run.trial082.stdout:Installed: 2026-02-22T22:55:36.038 INFO:teuthology.orchestra.run.trial082.stdout: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:36.038 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:36.038 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:36.065 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install librados2 2026-02-22T22:55:36.378 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:55 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:36.443 INFO:teuthology.orchestra.run.trial082.stdout:Package librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:36.461 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:36.462 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:36.462 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:36.483 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install librbd1 2026-02-22T22:55:36.790 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:55 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:36.854 INFO:teuthology.orchestra.run.trial082.stdout:Package librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:36.872 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:36.873 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:36.873 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:36.894 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install python3-rados 2026-02-22T22:55:37.210 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:56 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:37.275 INFO:teuthology.orchestra.run.trial082.stdout:Package python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:37.294 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:37.294 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:37.294 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:37.316 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install python3-rgw 2026-02-22T22:55:37.629 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:56 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:37.694 INFO:teuthology.orchestra.run.trial082.stdout:Package python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:37.713 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:37.713 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:37.714 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:37.735 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install python3-cephfs 2026-02-22T22:55:38.047 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:57 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:38.112 INFO:teuthology.orchestra.run.trial082.stdout:Package python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:38.130 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:38.131 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:38.131 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:38.151 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install python3-rbd 2026-02-22T22:55:38.463 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:57 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:38.528 INFO:teuthology.orchestra.run.trial082.stdout:Package python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:38.547 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:38.547 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:38.547 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:38.569 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install rbd-fuse 2026-02-22T22:55:38.877 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:57 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:38.960 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:38.961 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:38.961 INFO:teuthology.orchestra.run.trial082.stdout: Package Architecture Version Repository Size 2026-02-22T22:55:38.961 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:38.961 INFO:teuthology.orchestra.run.trial082.stdout:Installing: 2026-02-22T22:55:38.961 INFO:teuthology.orchestra.run.trial082.stdout: rbd-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 85 k 2026-02-22T22:55:38.961 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:38.961 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:55:38.961 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:38.962 INFO:teuthology.orchestra.run.trial082.stdout:Install 1 Package 2026-02-22T22:55:38.962 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:38.962 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 85 k 2026-02-22T22:55:38.962 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 227 k 2026-02-22T22:55:38.962 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:55:39.366 INFO:teuthology.orchestra.run.trial082.stdout:rbd-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 210 kB/s | 85 kB 00:00 2026-02-22T22:55:39.366 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:39.367 INFO:teuthology.orchestra.run.trial082.stdout:Total 210 kB/s | 85 kB 00:00 2026-02-22T22:55:39.367 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:55:39.370 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:55:39.370 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:55:39.401 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:55:39.401 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:55:39.451 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:55:39.468 INFO:teuthology.orchestra.run.trial082.stdout: Installing : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:39.548 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:39.606 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:39.606 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:39.606 INFO:teuthology.orchestra.run.trial082.stdout:Installed: 2026-02-22T22:55:39.606 INFO:teuthology.orchestra.run.trial082.stdout: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:39.606 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:39.606 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:39.634 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install rbd-mirror 2026-02-22T22:55:39.951 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:00:58 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:40.036 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:40.037 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:40.037 INFO:teuthology.orchestra.run.trial082.stdout: Package Arch Version Repo Size 2026-02-22T22:55:40.037 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:40.037 INFO:teuthology.orchestra.run.trial082.stdout:Installing: 2026-02-22T22:55:40.037 INFO:teuthology.orchestra.run.trial082.stdout: rbd-mirror x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.1 M 2026-02-22T22:55:40.037 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:40.037 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:55:40.037 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:40.037 INFO:teuthology.orchestra.run.trial082.stdout:Install 1 Package 2026-02-22T22:55:40.038 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:40.038 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 3.1 M 2026-02-22T22:55:40.038 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 13 M 2026-02-22T22:55:40.038 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:55:40.510 INFO:teuthology.orchestra.run.trial082.stdout:rbd-mirror-19.2.3-689.gf90f4eae.el9.x86_64.rpm 6.6 MB/s | 3.1 MB 00:00 2026-02-22T22:55:40.511 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:40.511 INFO:teuthology.orchestra.run.trial082.stdout:Total 6.6 MB/s | 3.1 MB 00:00 2026-02-22T22:55:40.511 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:55:40.515 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:55:40.515 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:55:40.555 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:55:40.556 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:55:40.749 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:55:40.751 INFO:teuthology.orchestra.run.trial082.stdout: Installing : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:40.773 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:40.773 INFO:teuthology.orchestra.run.trial082.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:55:40.774 INFO:teuthology.orchestra.run.trial082.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-22T22:55:40.774 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:55:40.774 INFO:teuthology.orchestra.run.trial082.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:55:40.774 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:41.065 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:41.065 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:41.065 INFO:teuthology.orchestra.run.trial082.stdout:Installed: 2026-02-22T22:55:41.065 INFO:teuthology.orchestra.run.trial082.stdout: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:41.065 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:41.065 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:41.093 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install rbd-nbd 2026-02-22T22:55:41.406 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:01:00 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:41.489 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:41.489 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:41.489 INFO:teuthology.orchestra.run.trial082.stdout: Package Architecture Version Repository Size 2026-02-22T22:55:41.489 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:41.489 INFO:teuthology.orchestra.run.trial082.stdout:Installing: 2026-02-22T22:55:41.490 INFO:teuthology.orchestra.run.trial082.stdout: rbd-nbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 170 k 2026-02-22T22:55:41.490 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:41.490 INFO:teuthology.orchestra.run.trial082.stdout:Transaction Summary 2026-02-22T22:55:41.490 INFO:teuthology.orchestra.run.trial082.stdout:================================================================================ 2026-02-22T22:55:41.490 INFO:teuthology.orchestra.run.trial082.stdout:Install 1 Package 2026-02-22T22:55:41.490 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:41.490 INFO:teuthology.orchestra.run.trial082.stdout:Total download size: 170 k 2026-02-22T22:55:41.490 INFO:teuthology.orchestra.run.trial082.stdout:Installed size: 490 k 2026-02-22T22:55:41.490 INFO:teuthology.orchestra.run.trial082.stdout:Downloading Packages: 2026-02-22T22:55:41.610 INFO:teuthology.orchestra.run.trial082.stdout:rbd-nbd-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.4 MB/s | 170 kB 00:00 2026-02-22T22:55:41.611 INFO:teuthology.orchestra.run.trial082.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:55:41.611 INFO:teuthology.orchestra.run.trial082.stdout:Total 1.4 MB/s | 170 kB 00:00 2026-02-22T22:55:41.611 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction check 2026-02-22T22:55:41.614 INFO:teuthology.orchestra.run.trial082.stdout:Transaction check succeeded. 2026-02-22T22:55:41.614 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction test 2026-02-22T22:55:41.645 INFO:teuthology.orchestra.run.trial082.stdout:Transaction test succeeded. 2026-02-22T22:55:41.645 INFO:teuthology.orchestra.run.trial082.stdout:Running transaction 2026-02-22T22:55:41.700 INFO:teuthology.orchestra.run.trial082.stdout: Preparing : 1/1 2026-02-22T22:55:41.718 INFO:teuthology.orchestra.run.trial082.stdout: Installing : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:41.800 INFO:teuthology.orchestra.run.trial082.stdout: Running scriptlet: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:41.858 INFO:teuthology.orchestra.run.trial082.stdout: Verifying : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:55:41.858 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:41.858 INFO:teuthology.orchestra.run.trial082.stdout:Installed: 2026-02-22T22:55:41.858 INFO:teuthology.orchestra.run.trial082.stdout: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:55:41.858 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:41.858 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:41.887 DEBUG:teuthology.orchestra.run.trial082:> sudo yum -y install rbd-mirror 2026-02-22T22:55:42.205 INFO:teuthology.orchestra.run.trial082.stdout:Last metadata expiration check: 0:01:01 ago on Sun 22 Feb 2026 10:54:41 PM UTC. 2026-02-22T22:55:42.271 INFO:teuthology.orchestra.run.trial082.stdout:Package rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:55:42.290 INFO:teuthology.orchestra.run.trial082.stdout:Dependencies resolved. 2026-02-22T22:55:42.290 INFO:teuthology.orchestra.run.trial082.stdout:Nothing to do. 2026-02-22T22:55:42.290 INFO:teuthology.orchestra.run.trial082.stdout:Complete! 2026-02-22T22:55:42.316 DEBUG:teuthology.parallel:result is None 2026-02-22T22:55:42.317 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:55:42.490 DEBUG:teuthology.orchestra.run.trial060:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-22T22:55:42.511 INFO:teuthology.orchestra.run.trial060.stdout:19.2.3-689.gf90f4eae.el9 2026-02-22T22:55:42.511 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689.gf90f4eae.el9 2026-02-22T22:55:42.511 INFO:teuthology.task.install:The correct ceph version 19.2.3-689.gf90f4eae is installed. 2026-02-22T22:55:42.514 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:55:42.673 DEBUG:teuthology.orchestra.run.trial082:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-22T22:55:42.694 INFO:teuthology.orchestra.run.trial082.stdout:19.2.3-689.gf90f4eae.el9 2026-02-22T22:55:42.694 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689.gf90f4eae.el9 2026-02-22T22:55:42.694 INFO:teuthology.task.install:The correct ceph version 19.2.3-689.gf90f4eae is installed. 2026-02-22T22:55:42.696 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-22T22:55:42.696 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:55:42.696 DEBUG:teuthology.orchestra.run.trial060:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T22:55:42.723 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:55:42.723 DEBUG:teuthology.orchestra.run.trial082:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T22:55:42.758 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-22T22:55:42.759 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:55:42.759 DEBUG:teuthology.orchestra.run.trial060:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T22:55:42.786 DEBUG:teuthology.orchestra.run.trial060:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T22:55:42.851 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:55:42.851 DEBUG:teuthology.orchestra.run.trial082:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T22:55:42.876 DEBUG:teuthology.orchestra.run.trial082:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T22:55:42.943 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-22T22:55:42.943 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:55:42.943 DEBUG:teuthology.orchestra.run.trial060:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T22:55:42.968 DEBUG:teuthology.orchestra.run.trial060:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T22:55:43.031 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:55:43.031 DEBUG:teuthology.orchestra.run.trial082:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T22:55:43.056 DEBUG:teuthology.orchestra.run.trial082:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T22:55:43.120 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-22T22:55:43.120 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:55:43.121 DEBUG:teuthology.orchestra.run.trial060:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T22:55:43.146 DEBUG:teuthology.orchestra.run.trial060:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T22:55:43.211 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:55:43.211 DEBUG:teuthology.orchestra.run.trial082:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T22:55:43.236 DEBUG:teuthology.orchestra.run.trial082:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T22:55:43.300 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T22:55:43.415 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-22T22:55:43.416 DEBUG:teuthology.orchestra.run.trial060:> sudo chmod 777 /var/log/ceph 2026-02-22T22:55:43.418 DEBUG:teuthology.orchestra.run.trial082:> sudo chmod 777 /var/log/ceph 2026-02-22T22:55:43.443 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-22T22:55:43.443 DEBUG:teuthology.orchestra.run.trial060:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T22:55:43.483 DEBUG:teuthology.orchestra.run.trial082:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T22:55:43.506 INFO:tasks.ceph:Creating extra log directories... 2026-02-22T22:55:43.506 DEBUG:teuthology.orchestra.run.trial060:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T22:55:43.550 DEBUG:teuthology.orchestra.run.trial082:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T22:55:43.598 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-22T22:55:43.598 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T22:55:43.599 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-bitmap supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-exclusive-lock}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64284', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial060.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYMkwFUQ6cXE+ZjCzk3UVcU7g01xRvGge6562s1gf58Hws4h6Io7AwM+0Sbn9MFfRklivaxmFE6mipfm0hckrE=', 'trial082.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPglXcHDxID8MMv+LLgiKjwWZtASzV0uUItaQbBbwe7w6JE63GHaCY+64iPiIcJLBXEE5mo+m1+LkfyxusecEPE='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock', 'RBD_MIRROR_CONFIG_KEY': '1', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T22:55:43.599 DEBUG:teuthology.orchestra.run.trial060:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T22:55:43.619 DEBUG:teuthology.orchestra.run.trial082:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T22:55:43.652 DEBUG:teuthology.orchestra.run.trial060:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:55:43.677 DEBUG:teuthology.orchestra.run.trial082:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:55:43.718 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:55:43.719 DEBUG:teuthology.orchestra.run.trial060:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T22:55:43.757 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T22:55:43.757 DEBUG:teuthology.orchestra.run.trial060:> stat /dev/vg_nvme/lv_1 2026-02-22T22:55:43.815 INFO:teuthology.orchestra.run.trial060.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T22:55:43.815 INFO:teuthology.orchestra.run.trial060.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:55:43.815 INFO:teuthology.orchestra.run.trial060.stdout:Device: 6h/6d Inode: 1299 Links: 1 2026-02-22T22:55:43.815 INFO:teuthology.orchestra.run.trial060.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:55:43.815 INFO:teuthology.orchestra.run.trial060.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:55:43.815 INFO:teuthology.orchestra.run.trial060.stdout:Access: 2026-02-22 22:55:23.030923598 +0000 2026-02-22T22:55:43.815 INFO:teuthology.orchestra.run.trial060.stdout:Modify: 2026-02-22 22:54:48.019671936 +0000 2026-02-22T22:55:43.815 INFO:teuthology.orchestra.run.trial060.stdout:Change: 2026-02-22 22:54:48.019671936 +0000 2026-02-22T22:55:43.816 INFO:teuthology.orchestra.run.trial060.stdout: Birth: 2026-02-22 22:54:48.019671936 +0000 2026-02-22T22:55:43.816 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T22:55:43.879 INFO:teuthology.orchestra.run.trial060.stderr:1+0 records in 2026-02-22T22:55:43.880 INFO:teuthology.orchestra.run.trial060.stderr:1+0 records out 2026-02-22T22:55:43.880 INFO:teuthology.orchestra.run.trial060.stderr:512 bytes copied, 0.000144639 s, 3.5 MB/s 2026-02-22T22:55:43.881 DEBUG:teuthology.orchestra.run.trial060:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T22:55:43.938 DEBUG:teuthology.orchestra.run.trial060:> stat /dev/vg_nvme/lv_2 2026-02-22T22:55:43.995 INFO:teuthology.orchestra.run.trial060.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T22:55:43.995 INFO:teuthology.orchestra.run.trial060.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:55:43.995 INFO:teuthology.orchestra.run.trial060.stdout:Device: 6h/6d Inode: 1305 Links: 1 2026-02-22T22:55:43.995 INFO:teuthology.orchestra.run.trial060.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:55:43.995 INFO:teuthology.orchestra.run.trial060.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:55:43.995 INFO:teuthology.orchestra.run.trial060.stdout:Access: 2026-02-22 22:55:23.031923577 +0000 2026-02-22T22:55:43.995 INFO:teuthology.orchestra.run.trial060.stdout:Modify: 2026-02-22 22:54:48.020671914 +0000 2026-02-22T22:55:43.995 INFO:teuthology.orchestra.run.trial060.stdout:Change: 2026-02-22 22:54:48.020671914 +0000 2026-02-22T22:55:43.995 INFO:teuthology.orchestra.run.trial060.stdout: Birth: 2026-02-22 22:54:48.020671914 +0000 2026-02-22T22:55:43.996 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T22:55:44.058 INFO:teuthology.orchestra.run.trial060.stderr:1+0 records in 2026-02-22T22:55:44.059 INFO:teuthology.orchestra.run.trial060.stderr:1+0 records out 2026-02-22T22:55:44.059 INFO:teuthology.orchestra.run.trial060.stderr:512 bytes copied, 0.000139078 s, 3.7 MB/s 2026-02-22T22:55:44.060 DEBUG:teuthology.orchestra.run.trial060:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T22:55:44.117 DEBUG:teuthology.orchestra.run.trial060:> stat /dev/vg_nvme/lv_3 2026-02-22T22:55:44.174 INFO:teuthology.orchestra.run.trial060.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T22:55:44.175 INFO:teuthology.orchestra.run.trial060.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:55:44.175 INFO:teuthology.orchestra.run.trial060.stdout:Device: 6h/6d Inode: 1312 Links: 1 2026-02-22T22:55:44.175 INFO:teuthology.orchestra.run.trial060.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:55:44.175 INFO:teuthology.orchestra.run.trial060.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:55:44.175 INFO:teuthology.orchestra.run.trial060.stdout:Access: 2026-02-22 22:55:23.031923577 +0000 2026-02-22T22:55:44.175 INFO:teuthology.orchestra.run.trial060.stdout:Modify: 2026-02-22 22:54:48.020671914 +0000 2026-02-22T22:55:44.175 INFO:teuthology.orchestra.run.trial060.stdout:Change: 2026-02-22 22:54:48.020671914 +0000 2026-02-22T22:55:44.175 INFO:teuthology.orchestra.run.trial060.stdout: Birth: 2026-02-22 22:54:48.020671914 +0000 2026-02-22T22:55:44.176 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T22:55:44.237 INFO:teuthology.orchestra.run.trial060.stderr:1+0 records in 2026-02-22T22:55:44.237 INFO:teuthology.orchestra.run.trial060.stderr:1+0 records out 2026-02-22T22:55:44.237 INFO:teuthology.orchestra.run.trial060.stderr:512 bytes copied, 0.000143306 s, 3.6 MB/s 2026-02-22T22:55:44.238 DEBUG:teuthology.orchestra.run.trial060:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T22:55:44.296 DEBUG:teuthology.orchestra.run.trial060:> stat /dev/vg_nvme/lv_4 2026-02-22T22:55:44.352 INFO:teuthology.orchestra.run.trial060.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T22:55:44.352 INFO:teuthology.orchestra.run.trial060.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:55:44.352 INFO:teuthology.orchestra.run.trial060.stdout:Device: 6h/6d Inode: 1300 Links: 1 2026-02-22T22:55:44.352 INFO:teuthology.orchestra.run.trial060.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:55:44.352 INFO:teuthology.orchestra.run.trial060.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:55:44.352 INFO:teuthology.orchestra.run.trial060.stdout:Access: 2026-02-22 22:55:23.031923577 +0000 2026-02-22T22:55:44.352 INFO:teuthology.orchestra.run.trial060.stdout:Modify: 2026-02-22 22:54:48.019671936 +0000 2026-02-22T22:55:44.352 INFO:teuthology.orchestra.run.trial060.stdout:Change: 2026-02-22 22:54:48.019671936 +0000 2026-02-22T22:55:44.353 INFO:teuthology.orchestra.run.trial060.stdout: Birth: 2026-02-22 22:54:48.019671936 +0000 2026-02-22T22:55:44.353 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T22:55:44.416 INFO:teuthology.orchestra.run.trial060.stderr:1+0 records in 2026-02-22T22:55:44.416 INFO:teuthology.orchestra.run.trial060.stderr:1+0 records out 2026-02-22T22:55:44.416 INFO:teuthology.orchestra.run.trial060.stderr:512 bytes copied, 0.000144178 s, 3.6 MB/s 2026-02-22T22:55:44.418 DEBUG:teuthology.orchestra.run.trial060:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T22:55:44.475 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:55:44.475 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial060.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:55:44.475 INFO:tasks.ceph:Generating config... 2026-02-22T22:55:44.478 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T22:55:44.478 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T22:55:44.478 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T22:55:44.478 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T22:55:44.478 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.478 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.478 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.479 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.480 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T22:55:44.480 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T22:55:44.480 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T22:55:44.480 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T22:55:44.480 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T22:55:44.480 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T22:55:44.480 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T22:55:44.480 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T22:55:44.480 INFO:tasks.ceph:[osd] bdev async discard = True 2026-02-22T22:55:44.480 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-02-22T22:55:44.480 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2026-02-22T22:55:44.480 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T22:55:44.481 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T22:55:44.482 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-22T22:55:44.482 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-22T22:55:44.549 INFO:teuthology.orchestra.run.trial060.stdout:creating /etc/ceph/cluster1.keyring 2026-02-22T22:55:44.552 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-22T22:55:44.629 DEBUG:teuthology.orchestra.run.trial060:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:55:44.692 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.60')] 2026-02-22T22:55:44.693 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.60', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T22:55:44.694 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:55:44.694 DEBUG:teuthology.orchestra.run.trial060:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:55:44.748 DEBUG:teuthology.orchestra.run.trial060:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.60 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:55:44.819 INFO:teuthology.orchestra.run.trial060.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:55:44.819 INFO:teuthology.orchestra.run.trial060.stdout:monmaptool: generated fsid c7d96b99-86c6-4a4e-9557-db334b970cfe 2026-02-22T22:55:44.820 INFO:teuthology.orchestra.run.trial060.stdout:setting min_mon_release = quincy 2026-02-22T22:55:44.820 INFO:teuthology.orchestra.run.trial060.stdout:epoch 0 2026-02-22T22:55:44.820 INFO:teuthology.orchestra.run.trial060.stdout:fsid c7d96b99-86c6-4a4e-9557-db334b970cfe 2026-02-22T22:55:44.820 INFO:teuthology.orchestra.run.trial060.stdout:last_changed 2026-02-22T22:55:44.820926+0000 2026-02-22T22:55:44.820 INFO:teuthology.orchestra.run.trial060.stdout:created 2026-02-22T22:55:44.820926+0000 2026-02-22T22:55:44.820 INFO:teuthology.orchestra.run.trial060.stdout:min_mon_release 17 (quincy) 2026-02-22T22:55:44.820 INFO:teuthology.orchestra.run.trial060.stdout:election_strategy: 1 2026-02-22T22:55:44.820 INFO:teuthology.orchestra.run.trial060.stdout:0: [v2:10.20.193.60:3300/0,v1:10.20.193.60:6789/0] mon.a 2026-02-22T22:55:44.820 INFO:teuthology.orchestra.run.trial060.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-22T22:55:44.822 DEBUG:teuthology.orchestra.run.trial060:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:55:44.878 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID c7d96b99-86c6-4a4e-9557-db334b970cfe... 2026-02-22T22:55:44.880 DEBUG:teuthology.orchestra.run.trial060:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T22:55:44.920 DEBUG:teuthology.orchestra.run.trial082:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T22:55:44.960 INFO:teuthology.orchestra.run.trial060.stdout:[global] 2026-02-22T22:55:44.960 INFO:teuthology.orchestra.run.trial060.stdout: chdir = "" 2026-02-22T22:55:44.960 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:55:44.960 INFO:teuthology.orchestra.run.trial060.stdout: auth supported = cephx 2026-02-22T22:55:44.960 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.960 INFO:teuthology.orchestra.run.trial060.stdout: filestore xattr use omap = true 2026-02-22T22:55:44.960 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.960 INFO:teuthology.orchestra.run.trial060.stdout: mon clock drift allowed = 1.000 2026-02-22T22:55:44.960 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: osd crush chooseleaf type = 0 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: auth debug = true 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: ms die on old message = true 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: ms die on bug = true 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: mon pg warn max object skew = 0 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:55:44.961 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: osd pool default size = 2 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: mon osd allow primary affinity = true 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: mon osd allow pg remap = true 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on no sortbitwise = false 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on too few osds = false 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:55:44.962 INFO:teuthology.orchestra.run.trial060.stdout: mon_allow_pool_size_one = true 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: osd default data pool replay window = 5 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: mon allow pool delete = true 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: mon cluster log file level = debug 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: debug asserts on shutdown = true 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: mon health detail to clog = false 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: mon host = 10.20.193.60 2026-02-22T22:55:44.963 INFO:teuthology.orchestra.run.trial060.stdout: mon client directed command retry = 5 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on pool no app = False 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: ms inject socket failures = 5000 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: fsid = c7d96b99-86c6-4a4e-9557-db334b970cfe 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout:[osd] 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: osd journal size = 100 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: osd scrub load threshold = 5.0 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: osd scrub max interval = 600 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: osd mclock skip benchmark = true 2026-02-22T22:55:44.964 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: osd recover clone overlap = true 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: osd recovery max chunk = 1048576 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: osd debug shutdown = true 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: osd debug op order = true 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: osd debug verify stray on activate = true 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: osd debug trim objects = true 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: osd open classes on start = true 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: osd debug pg log writeout = true 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:55:44.965 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: osd map max advance = 10 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: journal zero on create = true 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: filestore apply finisher threads = 3 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: bdev debug aio = true 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: osd debug misdirected ops = true 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: bdev async discard = True 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: bdev enable discard = True 2026-02-22T22:55:44.966 INFO:teuthology.orchestra.run.trial060.stdout: bluestore allocator = bitmap 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: bluestore block size = 96636764160 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: bluestore fsck on mount = True 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: debug bluefs = 1/20 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: debug bluestore = 1/20 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: debug ms = 1 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: debug osd = 20 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: debug rocksdb = 4/10 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: mon osd full ratio = 0.9 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: osd objectstore = bluestore 2026-02-22T22:55:44.967 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout:[mgr] 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: debug ms = 1 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: debug mgr = 20 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: debug mon = 20 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: debug auth = 20 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: mgr/telemetry/nag = false 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout:[mon] 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: debug ms = 1 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: debug mon = 20 2026-02-22T22:55:44.968 INFO:teuthology.orchestra.run.trial060.stdout: debug paxos = 20 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: debug auth = 20 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: mon data avail warn = 5 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: mon mgr mkfs grace = 240 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: mon osd reporter subtree level = osd 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: mon osd prime pg temp = true 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:55:44.969 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: # 1m isn't quite enough 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout:[client] 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: rgw cache enabled = true 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: rgw enable ops log = true 2026-02-22T22:55:44.970 INFO:teuthology.orchestra.run.trial060.stdout: rgw enable usage log = true 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout: debug rbd = 20 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout: debug rbd_mirror = 15 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout: log to stderr = False 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout: rbd default features = 125 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout:[mon.a] 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.0] 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.1] 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.971 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.2] 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.3] 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.4] 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.5] 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.972 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.6] 2026-02-22T22:55:44.973 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.973 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.974 INFO:teuthology.orchestra.run.trial082.stdout:[global] 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: chdir = "" 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: auth supported = cephx 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: filestore xattr use omap = true 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: mon clock drift allowed = 1.000 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: osd crush chooseleaf type = 0 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: auth debug = true 2026-02-22T22:55:44.975 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: ms die on old message = true 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: ms die on bug = true 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: mon pg warn max object skew = 0 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: osd pool default size = 2 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.976 INFO:teuthology.orchestra.run.trial082.stdout: mon osd allow primary affinity = true 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: mon osd allow pg remap = true 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on no sortbitwise = false 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on too few osds = false 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: mon_allow_pool_size_one = true 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.977 INFO:teuthology.orchestra.run.trial082.stdout: osd default data pool replay window = 5 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: mon allow pool delete = true 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: mon cluster log file level = debug 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: debug asserts on shutdown = true 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: mon health detail to clog = false 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: mon host = 10.20.193.60 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: mon client directed command retry = 5 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on pool no app = False 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: ms inject socket failures = 5000 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: fsid = c7d96b99-86c6-4a4e-9557-db334b970cfe 2026-02-22T22:55:44.978 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout:[osd] 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: osd journal size = 100 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: osd scrub load threshold = 5.0 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: osd scrub max interval = 600 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: osd mclock skip benchmark = true 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: osd recover clone overlap = true 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: osd recovery max chunk = 1048576 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: osd debug shutdown = true 2026-02-22T22:55:44.979 INFO:teuthology.orchestra.run.trial082.stdout: osd debug op order = true 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: osd debug verify stray on activate = true 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: osd debug trim objects = true 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: osd open classes on start = true 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: osd debug pg log writeout = true 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: osd map max advance = 10 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: journal zero on create = true 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.980 INFO:teuthology.orchestra.run.trial082.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: filestore apply finisher threads = 3 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: bdev debug aio = true 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: osd debug misdirected ops = true 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: bdev async discard = True 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: bdev enable discard = True 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: bluestore allocator = bitmap 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: bluestore block size = 96636764160 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: bluestore fsck on mount = True 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: debug bluefs = 1/20 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: debug bluestore = 1/20 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: debug ms = 1 2026-02-22T22:55:44.981 INFO:teuthology.orchestra.run.trial082.stdout: debug osd = 20 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: debug rocksdb = 4/10 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: mon osd full ratio = 0.9 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: osd objectstore = bluestore 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout:[mgr] 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: debug ms = 1 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: debug mgr = 20 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: debug mon = 20 2026-02-22T22:55:44.982 INFO:teuthology.orchestra.run.trial082.stdout: debug auth = 20 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: mgr/telemetry/nag = false 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout:[mon] 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: debug ms = 1 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: debug mon = 20 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: debug paxos = 20 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: debug auth = 20 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: mon data avail warn = 5 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: mon mgr mkfs grace = 240 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:55:44.983 INFO:teuthology.orchestra.run.trial082.stdout: mon osd reporter subtree level = osd 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: mon osd prime pg temp = true 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: # 1m isn't quite enough 2026-02-22T22:55:44.984 INFO:teuthology.orchestra.run.trial082.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout:[client] 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: rgw cache enabled = true 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: rgw enable ops log = true 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: rgw enable usage log = true 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: debug rbd = 20 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: debug rbd_mirror = 15 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: log to stderr = False 2026-02-22T22:55:44.985 INFO:teuthology.orchestra.run.trial082.stdout: rbd default features = 125 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout:[mon.a] 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.0] 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.1] 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.2] 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.3] 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.986 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.987 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.4] 2026-02-22T22:55:44.987 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.987 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.987 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.5] 2026-02-22T22:55:44.987 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.987 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.987 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.6] 2026-02-22T22:55:44.987 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:55:44.987 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:55:44.987 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-22T22:55:44.988 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-22T22:55:45.056 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T22:55:45.056 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:55:45.056 DEBUG:teuthology.orchestra.run.trial060:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-22T22:55:45.112 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:55:45.112 DEBUG:teuthology.orchestra.run.trial060:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-22T22:55:45.169 INFO:tasks.ceph:Sending monmap to node ubuntu@trial060.front.sepia.ceph.com 2026-02-22T22:55:45.169 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:55:45.169 DEBUG:teuthology.orchestra.run.trial060:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T22:55:45.169 DEBUG:teuthology.orchestra.run.trial060:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:55:45.242 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:55:45.242 DEBUG:teuthology.orchestra.run.trial060:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:55:45.300 INFO:tasks.ceph:Sending monmap to node ubuntu@trial082.front.sepia.ceph.com 2026-02-22T22:55:45.301 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:55:45.301 DEBUG:teuthology.orchestra.run.trial082:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T22:55:45.301 DEBUG:teuthology.orchestra.run.trial082:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:55:45.334 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:55:45.335 DEBUG:teuthology.orchestra.run.trial082:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:55:45.391 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T22:55:45.391 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T22:55:45.391 DEBUG:teuthology.orchestra.run.trial060:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T22:55:45.435 INFO:teuthology.orchestra.run.trial060.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T22:55:45.438 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T22:55:45.438 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T22:55:45.439 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-22T22:55:45.511 INFO:teuthology.orchestra.run.trial060.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-22T22:55:45.523 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T22:55:45.559 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T22:55:45.570 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T22:55:45.645 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T22:55:45.656 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T22:55:45.731 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T22:55:45.742 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T22:55:45.815 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T22:55:45.826 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T22:55:45.901 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T22:55:45.913 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T22:55:45.987 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T22:55:45.998 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T22:55:46.072 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T22:55:46.083 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T22:55:46.156 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T22:55:46.166 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T22:55:46.167 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial060.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:55:46.167 DEBUG:teuthology.orchestra.run.trial060:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-22T22:55:46.193 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:55:46.193 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-22T22:55:46.194 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial060.front.sepia.ceph.com 2026-02-22T22:55:46.194 DEBUG:teuthology.orchestra.run.trial060:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T22:55:46.259 INFO:teuthology.orchestra.run.trial060.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:55:46.259 INFO:teuthology.orchestra.run.trial060.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:55:46.260 INFO:teuthology.orchestra.run.trial060.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:55:46.260 INFO:teuthology.orchestra.run.trial060.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:55:46.260 INFO:teuthology.orchestra.run.trial060.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:55:46.260 INFO:teuthology.orchestra.run.trial060.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:55:46.260 INFO:teuthology.orchestra.run.trial060.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:55:46.260 INFO:teuthology.orchestra.run.trial060.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:55:46.260 INFO:teuthology.orchestra.run.trial060.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:55:46.260 INFO:teuthology.orchestra.run.trial060.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:55:46.836 INFO:teuthology.orchestra.run.trial060.stdout:Discarding blocks...Done. 2026-02-22T22:55:46.839 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial060.front.sepia.ceph.com -o noatime 2026-02-22T22:55:46.839 DEBUG:teuthology.orchestra.run.trial060:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-22T22:55:46.906 DEBUG:teuthology.orchestra.run.trial060:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-22T22:55:46.974 DEBUG:teuthology.orchestra.run.trial060:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-22T22:55:47.038 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:55:47.038 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-22T22:55:47.039 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial060.front.sepia.ceph.com 2026-02-22T22:55:47.039 DEBUG:teuthology.orchestra.run.trial060:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T22:55:47.104 INFO:teuthology.orchestra.run.trial060.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:55:47.104 INFO:teuthology.orchestra.run.trial060.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:55:47.104 INFO:teuthology.orchestra.run.trial060.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:55:47.104 INFO:teuthology.orchestra.run.trial060.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:55:47.105 INFO:teuthology.orchestra.run.trial060.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:55:47.105 INFO:teuthology.orchestra.run.trial060.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:55:47.105 INFO:teuthology.orchestra.run.trial060.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:55:47.105 INFO:teuthology.orchestra.run.trial060.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:55:47.105 INFO:teuthology.orchestra.run.trial060.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:55:47.105 INFO:teuthology.orchestra.run.trial060.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:55:47.668 INFO:teuthology.orchestra.run.trial060.stdout:Discarding blocks...Done. 2026-02-22T22:55:47.670 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial060.front.sepia.ceph.com -o noatime 2026-02-22T22:55:47.670 DEBUG:teuthology.orchestra.run.trial060:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-22T22:55:47.738 DEBUG:teuthology.orchestra.run.trial060:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-22T22:55:47.805 DEBUG:teuthology.orchestra.run.trial060:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-22T22:55:47.868 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:55:47.868 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-22T22:55:47.869 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial060.front.sepia.ceph.com 2026-02-22T22:55:47.869 DEBUG:teuthology.orchestra.run.trial060:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T22:55:47.933 INFO:teuthology.orchestra.run.trial060.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:55:47.933 INFO:teuthology.orchestra.run.trial060.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:55:47.933 INFO:teuthology.orchestra.run.trial060.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:55:47.934 INFO:teuthology.orchestra.run.trial060.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:55:47.934 INFO:teuthology.orchestra.run.trial060.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:55:47.934 INFO:teuthology.orchestra.run.trial060.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:55:47.934 INFO:teuthology.orchestra.run.trial060.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:55:47.934 INFO:teuthology.orchestra.run.trial060.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:55:47.934 INFO:teuthology.orchestra.run.trial060.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:55:47.934 INFO:teuthology.orchestra.run.trial060.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:55:48.493 INFO:teuthology.orchestra.run.trial060.stdout:Discarding blocks...Done. 2026-02-22T22:55:48.496 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial060.front.sepia.ceph.com -o noatime 2026-02-22T22:55:48.496 DEBUG:teuthology.orchestra.run.trial060:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-22T22:55:48.569 DEBUG:teuthology.orchestra.run.trial060:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-22T22:55:48.636 DEBUG:teuthology.orchestra.run.trial060:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:55:48.714 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:48.715+0000 7f54881a7780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-22T22:55:48.715 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:48.715+0000 7f54881a7780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-22T22:55:48.715 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:48.716+0000 7f54881a7780 -1 bdev(0x55e7ad3c9800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-22T22:55:48.716 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:48.716+0000 7f54881a7780 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-22T22:55:52.604 DEBUG:teuthology.orchestra.run.trial060:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-22T22:55:52.669 DEBUG:teuthology.orchestra.run.trial060:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:55:52.751 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:52.752+0000 7f163b1d2780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-22T22:55:52.751 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:52.752+0000 7f163b1d2780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-22T22:55:52.751 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:52.752+0000 7f163b1d2780 -1 bdev(0x564e6a60f800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-22T22:55:52.751 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:52.752+0000 7f163b1d2780 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-22T22:55:56.645 DEBUG:teuthology.orchestra.run.trial060:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-22T22:55:56.709 DEBUG:teuthology.orchestra.run.trial060:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:55:56.787 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:56.787+0000 7fb71a109780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-22T22:55:56.787 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:56.787+0000 7fb71a109780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-22T22:55:56.787 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:56.787+0000 7fb71a109780 -1 bdev(0x55944da3d800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-22T22:55:56.787 INFO:teuthology.orchestra.run.trial060.stderr:2026-02-22T22:55:56.788+0000 7fb71a109780 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-22T22:56:00.675 DEBUG:teuthology.orchestra.run.trial060:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-22T22:56:00.740 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T22:56:00.747 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:00.747 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-22T22:56:00.805 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:00.805 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-22T22:56:00.875 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:00.875 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-22T22:56:00.946 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:00.947 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-22T22:56:01.017 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:01.017 DEBUG:teuthology.orchestra.run.trial060:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-22T22:56:01.074 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:01.075 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-22T22:56:01.094 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:01.094 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-22T22:56:01.150 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:01.150 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-22T22:56:01.208 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:01.208 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-22T22:56:01.265 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:01.265 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-22T22:56:01.323 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:01.323 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-22T22:56:01.379 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:01.379 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-22T22:56:01.435 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:01.435 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-22T22:56:01.492 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T22:56:01.492 DEBUG:teuthology.orchestra.run.trial060:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-22T22:56:01.516 INFO:teuthology.orchestra.run.trial060.stdout:[mgr.x] 2026-02-22T22:56:01.517 INFO:teuthology.orchestra.run.trial060.stdout: key = AQBxiZtpveYKGhAA0pHAfz99a6ssL5F2Z1DXuw== 2026-02-22T22:56:01.517 INFO:teuthology.orchestra.run.trial060.stdout:[osd.0] 2026-02-22T22:56:01.517 INFO:teuthology.orchestra.run.trial060.stdout: key = AQB0iZtp8v6wKhAACgyHvB90O2q6xhGknEVKqw== 2026-02-22T22:56:01.517 INFO:teuthology.orchestra.run.trial060.stdout:[osd.1] 2026-02-22T22:56:01.518 INFO:teuthology.orchestra.run.trial060.stdout: key = AQB4iZtpfl/cLBAAaN2RWrhr0wmBQG22aZPfHQ== 2026-02-22T22:56:01.518 INFO:teuthology.orchestra.run.trial060.stdout:[osd.2] 2026-02-22T22:56:01.518 INFO:teuthology.orchestra.run.trial060.stdout: key = AQB8iZtp5BL8LhAAP4akO2z9506ctYbIwPgX/g== 2026-02-22T22:56:01.518 INFO:teuthology.orchestra.run.trial060.stdout:[client.0] 2026-02-22T22:56:01.518 INFO:teuthology.orchestra.run.trial060.stdout: key = AQBxiZtppVuTHhAA1G/P5yJ7lNCZSNfjwANGYw== 2026-02-22T22:56:01.518 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror] 2026-02-22T22:56:01.519 INFO:teuthology.orchestra.run.trial060.stdout: key = AQBxiZtpWDhEIRAA9LzFBw6JUgRNP8NIbjBvdg== 2026-02-22T22:56:01.519 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.0] 2026-02-22T22:56:01.519 INFO:teuthology.orchestra.run.trial060.stdout: key = AQBxiZtpTpJiJhAAn1EVPKVZYet8RQ2sL3y3Zw== 2026-02-22T22:56:01.519 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.1] 2026-02-22T22:56:01.519 INFO:teuthology.orchestra.run.trial060.stdout: key = AQBxiZtpHjiDKxAACe6nwhQ9rgCwtiZ22bXINQ== 2026-02-22T22:56:01.520 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.2] 2026-02-22T22:56:01.520 INFO:teuthology.orchestra.run.trial060.stdout: key = AQBxiZtpOIePMBAAN40+eK2x5napvEudATk2PQ== 2026-02-22T22:56:01.520 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.3] 2026-02-22T22:56:01.520 INFO:teuthology.orchestra.run.trial060.stdout: key = AQBxiZtp7u6xNRAAe8S0enaPsfdlKELGKMMQkg== 2026-02-22T22:56:01.520 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.4] 2026-02-22T22:56:01.520 INFO:teuthology.orchestra.run.trial060.stdout: key = AQBxiZtpEsHJOhAAL8tq3E1130CW24LT6sIsFw== 2026-02-22T22:56:01.520 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.5] 2026-02-22T22:56:01.521 INFO:teuthology.orchestra.run.trial060.stdout: key = AQByiZtp4ig8BBAAG5d4FQ87VlosT+pcoJAksw== 2026-02-22T22:56:01.521 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.6] 2026-02-22T22:56:01.521 INFO:teuthology.orchestra.run.trial060.stdout: key = AQByiZtp9ZtFCRAAxggF6Tys+P6+KizI+pZpXw== 2026-02-22T22:56:01.523 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T22:56:01.603 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:56:01.681 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:56:01.763 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:56:01.840 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:01.923 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:02.004 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:02.084 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:02.165 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:02.245 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:02.327 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:02.410 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:02.491 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:02.570 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T22:56:02.570 DEBUG:teuthology.orchestra.run.trial060:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-22T22:56:02.633 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-22T22:56:02.717 DEBUG:teuthology.orchestra.run.trial060:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-22T22:56:02.740 DEBUG:teuthology.orchestra.run.trial060:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:56:02.796 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-22T22:56:02.797 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-22T22:56:02.797 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-22T22:56:02.839 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-22T22:56:02.839 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-22T22:56:02.839 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-22T22:56:02.840 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-22T22:56:02.842 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-22T22:56:02.842 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T22:56:02.843 DEBUG:teuthology.orchestra.run.trial060:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-22T22:56:02.943 INFO:teuthology.orchestra.run.trial060.stderr:adjusted tunables profile to default 2026-02-22T22:56:02.951 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T22:56:02.951 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-22T22:56:02.951 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:02.951 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-22T22:56:02.976 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:02.976 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-22T22:56:03.040 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:03.041 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-22T22:56:03.104 DEBUG:teuthology.orchestra.run.trial060:> sudo ceph --cluster cluster1 osd new f255f4c8-739d-4bd1-b589-bc63ac762e21 0 2026-02-22T22:56:03.246 INFO:teuthology.orchestra.run.trial060.stdout:0 2026-02-22T22:56:03.253 DEBUG:teuthology.orchestra.run.trial060:> sudo ceph --cluster cluster1 osd new 6ebac1ed-ac24-4619-94c0-c4cfe9c29d08 1 2026-02-22T22:56:03.354 INFO:teuthology.orchestra.run.trial060.stdout:1 2026-02-22T22:56:03.362 DEBUG:teuthology.orchestra.run.trial060:> sudo ceph --cluster cluster1 osd new 30a74eef-60f3-466e-8be5-b6e89425fb11 2 2026-02-22T22:56:03.455 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.455+0000 7fe50202d140 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T22:56:03.465 INFO:teuthology.orchestra.run.trial060.stdout:2 2026-02-22T22:56:03.472 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-22T22:56:03.472 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-22T22:56:03.475 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-22T22:56:03.475 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-22T22:56:03.475 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-22T22:56:03.478 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-22T22:56:03.478 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-22T22:56:03.478 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-22T22:56:03.480 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-22T22:56:03.481 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:56:03.580 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:03.580 INFO:teuthology.orchestra.run.trial060.stdout:{"epoch":5,"fsid":"c7d96b99-86c6-4a4e-9557-db334b970cfe","created":"2026-02-22T22:56:02.902367+0000","modified":"2026-02-22T22:56:03.466085+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:56:03.466085+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f255f4c8-739d-4bd1-b589-bc63ac762e21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"6ebac1ed-ac24-4619-94c0-c4cfe9c29d08","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"30a74eef-60f3-466e-8be5-b6e89425fb11","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:56:03.581 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.581+0000 7fe50202d140 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T22:56:03.588 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-22T22:56:03.588 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T22:56:03.608 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.608+0000 7fe50202d140 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T22:56:03.640 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.641+0000 7fe50202d140 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T22:56:03.663 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.664+0000 7fe50202d140 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T22:56:03.690 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.690+0000 7fe50202d140 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T22:56:03.745 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.746+0000 7fe50202d140 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T22:56:03.806 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T22:56:03.806 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T22:56:03.806 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr: from numpy import show_config as show_numpy_config 2026-02-22T22:56:03.809 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.810+0000 7fe50202d140 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T22:56:03.835 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.836+0000 7fe50202d140 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T22:56:03.890 DEBUG:teuthology.orchestra.run.trial060:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:56:03.891 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.891+0000 7fe50202d140 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2026-02-22T22:56:03.933 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.934+0000 7fe50202d140 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T22:56:03.959 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.960+0000 7fe50202d140 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T22:56:03.983 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:03.984+0000 7fe50202d140 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T22:56:03.985 INFO:teuthology.misc.health.trial060.stdout: 2026-02-22T22:56:03.985 INFO:teuthology.misc.health.trial060.stdout:{"epoch":5,"fsid":"c7d96b99-86c6-4a4e-9557-db334b970cfe","created":"2026-02-22T22:56:02.902367+0000","modified":"2026-02-22T22:56:03.466085+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:56:03.466085+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f255f4c8-739d-4bd1-b589-bc63ac762e21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"6ebac1ed-ac24-4619-94c0-c4cfe9c29d08","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"30a74eef-60f3-466e-8be5-b6e89425fb11","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:56:03.992 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:56:04.009 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.010+0000 7fe50202d140 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T22:56:04.032 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.032+0000 7fe50202d140 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T22:56:04.225 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.225+0000 7fe50202d140 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2026-02-22T22:56:04.279 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.280+0000 7fe50202d140 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T22:56:04.301 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.302+0000 7fe50202d140 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T22:56:04.324 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.325+0000 7fe50202d140 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T22:56:04.378 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.378+0000 7fe50202d140 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:56:04.398 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.399+0000 7fe50202d140 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T22:56:04.450 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.450+0000 7fe50202d140 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:56:04.847 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.847+0000 7fe50202d140 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T22:56:04.869 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:04.869+0000 7fe50202d140 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T22:56:05.210 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:05.210+0000 7fe50202d140 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T22:56:05.399 INFO:tasks.ceph.cluster1.osd.0.trial060.stderr:2026-02-22T22:56:05.399+0000 7f180620d780 -1 Falling back to public interface 2026-02-22T22:56:05.406 INFO:tasks.ceph.cluster1.osd.2.trial060.stderr:2026-02-22T22:56:05.407+0000 7f15b3b09780 -1 Falling back to public interface 2026-02-22T22:56:05.410 INFO:tasks.ceph.cluster1.osd.1.trial060.stderr:2026-02-22T22:56:05.411+0000 7fdca423f780 -1 Falling back to public interface 2026-02-22T22:56:06.214 INFO:tasks.ceph.cluster1.mgr.x.trial060.stderr:2026-02-22T22:56:06.214+0000 7fe41e53d640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T22:56:09.855 INFO:tasks.ceph.cluster1.osd.0.trial060.stderr:2026-02-22T22:56:09.855+0000 7f180620d780 -1 osd.0 0 log_to_monitors true 2026-02-22T22:56:09.907 INFO:tasks.ceph.cluster1.osd.2.trial060.stderr:2026-02-22T22:56:09.907+0000 7f15b3b09780 -1 osd.2 0 log_to_monitors true 2026-02-22T22:56:09.946 INFO:tasks.ceph.cluster1.osd.1.trial060.stderr:2026-02-22T22:56:09.945+0000 7fdca423f780 -1 osd.1 0 log_to_monitors true 2026-02-22T22:56:10.296 DEBUG:teuthology.orchestra.run.trial060:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:56:10.448 INFO:teuthology.misc.health.trial060.stdout: 2026-02-22T22:56:10.449 INFO:teuthology.misc.health.trial060.stdout:{"epoch":6,"fsid":"c7d96b99-86c6-4a4e-9557-db334b970cfe","created":"2026-02-22T22:56:02.902367+0000","modified":"2026-02-22T22:56:09.909153+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:56:03.466085+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f255f4c8-739d-4bd1-b589-bc63ac762e21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"6ebac1ed-ac24-4619-94c0-c4cfe9c29d08","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"30a74eef-60f3-466e-8be5-b6e89425fb11","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:56:10.456 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:56:10.921 INFO:tasks.ceph.cluster1.osd.0.trial060.stderr:2026-02-22T22:56:10.921+0000 7f1801a4a640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T22:56:10.923 INFO:tasks.ceph.cluster1.osd.0.trial060.stderr:2026-02-22T22:56:10.923+0000 7f17f987d640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:56:11.923 INFO:tasks.ceph.cluster1.osd.2.trial060.stderr:2026-02-22T22:56:11.923+0000 7f15afa74640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T22:56:11.924 INFO:tasks.ceph.cluster1.osd.1.trial060.stderr:2026-02-22T22:56:11.924+0000 7fdca01ac640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T22:56:11.924 INFO:tasks.ceph.cluster1.osd.2.trial060.stderr:2026-02-22T22:56:11.924+0000 7f15a78a7640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:56:11.926 INFO:tasks.ceph.cluster1.osd.1.trial060.stderr:2026-02-22T22:56:11.926+0000 7fdc97fdf640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:56:16.759 DEBUG:teuthology.orchestra.run.trial060:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:56:16.911 INFO:teuthology.misc.health.trial060.stdout: 2026-02-22T22:56:16.911 INFO:teuthology.misc.health.trial060.stdout:{"epoch":12,"fsid":"c7d96b99-86c6-4a4e-9557-db334b970cfe","created":"2026-02-22T22:56:02.902367+0000","modified":"2026-02-22T22:56:15.919720+0000","last_up_change":"2026-02-22T22:56:12.915552+0000","last_in_change":"2026-02-22T22:56:03.466085+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:56:13.226544+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f255f4c8-739d-4bd1-b589-bc63ac762e21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6802","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6803","nonce":1437422725}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6804","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6805","nonce":1437422725}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6808","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6809","nonce":1437422725}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6806","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6807","nonce":1437422725}]},"public_addr":"10.20.193.60:6803/1437422725","cluster_addr":"10.20.193.60:6805/1437422725","heartbeat_back_addr":"10.20.193.60:6809/1437422725","heartbeat_front_addr":"10.20.193.60:6807/1437422725","state":["exists","up"]},{"osd":1,"uuid":"6ebac1ed-ac24-4619-94c0-c4cfe9c29d08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6818","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6819","nonce":3794669997}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6820","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6821","nonce":3794669997}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6824","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6825","nonce":3794669997}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6822","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6823","nonce":3794669997}]},"public_addr":"10.20.193.60:6819/3794669997","cluster_addr":"10.20.193.60:6821/3794669997","heartbeat_back_addr":"10.20.193.60:6825/3794669997","heartbeat_front_addr":"10.20.193.60:6823/3794669997","state":["exists","up"]},{"osd":2,"uuid":"30a74eef-60f3-466e-8be5-b6e89425fb11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6810","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6811","nonce":4208307478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6812","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6813","nonce":4208307478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6816","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6817","nonce":4208307478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6814","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6815","nonce":4208307478}]},"public_addr":"10.20.193.60:6811/4208307478","cluster_addr":"10.20.193.60:6813/4208307478","heartbeat_back_addr":"10.20.193.60:6817/4208307478","heartbeat_front_addr":"10.20.193.60:6815/4208307478","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.840981+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.942404+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.876837+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:56:16.918 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T22:56:16.918 INFO:tasks.ceph:Creating RBD pool 2026-02-22T22:56:16.918 DEBUG:teuthology.orchestra.run.trial060:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-22T22:56:17.925 INFO:teuthology.orchestra.run.trial060.stderr:pool 'rbd' created 2026-02-22T22:56:17.936 DEBUG:teuthology.orchestra.run.trial060:> rbd --cluster cluster1 pool init rbd 2026-02-22T22:56:20.937 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-22T22:56:20.938 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-22T22:56:20.939 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-22T22:56:20.939 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-22T22:56:20.939 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T22:56:21.122 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:21.129 INFO:teuthology.orchestra.run.trial060.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6800","nonce":2275686875},{"type":"v1","addr":"10.20.193.60:6801","nonce":2275686875}]},"active_addr":"10.20.193.60:6801/2275686875","active_change":"2026-02-22T22:56:05.212328+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.60:0","nonce":148274440}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.60:0","nonce":3931263218}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.60:0","nonce":138892832}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.60:0","nonce":893558384}]}]} 2026-02-22T22:56:21.131 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-22T22:56:21.131 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-22T22:56:21.131 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:56:21.286 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:21.287 INFO:teuthology.orchestra.run.trial060.stdout:{"epoch":16,"fsid":"c7d96b99-86c6-4a4e-9557-db334b970cfe","created":"2026-02-22T22:56:02.902367+0000","modified":"2026-02-22T22:56:20.930282+0000","last_up_change":"2026-02-22T22:56:12.915552+0000","last_in_change":"2026-02-22T22:56:03.466085+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:56:13.226544+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:56:17.081581+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f255f4c8-739d-4bd1-b589-bc63ac762e21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6802","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6803","nonce":1437422725}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6804","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6805","nonce":1437422725}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6808","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6809","nonce":1437422725}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6806","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6807","nonce":1437422725}]},"public_addr":"10.20.193.60:6803/1437422725","cluster_addr":"10.20.193.60:6805/1437422725","heartbeat_back_addr":"10.20.193.60:6809/1437422725","heartbeat_front_addr":"10.20.193.60:6807/1437422725","state":["exists","up"]},{"osd":1,"uuid":"6ebac1ed-ac24-4619-94c0-c4cfe9c29d08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6818","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6819","nonce":3794669997}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6820","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6821","nonce":3794669997}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6824","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6825","nonce":3794669997}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6822","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6823","nonce":3794669997}]},"public_addr":"10.20.193.60:6819/3794669997","cluster_addr":"10.20.193.60:6821/3794669997","heartbeat_back_addr":"10.20.193.60:6825/3794669997","heartbeat_front_addr":"10.20.193.60:6823/3794669997","state":["exists","up"]},{"osd":2,"uuid":"30a74eef-60f3-466e-8be5-b6e89425fb11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6810","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6811","nonce":4208307478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6812","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6813","nonce":4208307478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6816","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6817","nonce":4208307478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6814","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6815","nonce":4208307478}]},"public_addr":"10.20.193.60:6811/4208307478","cluster_addr":"10.20.193.60:6813/4208307478","heartbeat_back_addr":"10.20.193.60:6817/4208307478","heartbeat_front_addr":"10.20.193.60:6815/4208307478","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.840981+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.942404+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.876837+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:56:21.295 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-22T22:56:21.295 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:56:21.454 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:21.454 INFO:teuthology.orchestra.run.trial060.stdout:{"epoch":16,"fsid":"c7d96b99-86c6-4a4e-9557-db334b970cfe","created":"2026-02-22T22:56:02.902367+0000","modified":"2026-02-22T22:56:20.930282+0000","last_up_change":"2026-02-22T22:56:12.915552+0000","last_in_change":"2026-02-22T22:56:03.466085+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:56:13.226544+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:56:17.081581+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f255f4c8-739d-4bd1-b589-bc63ac762e21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6802","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6803","nonce":1437422725}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6804","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6805","nonce":1437422725}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6808","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6809","nonce":1437422725}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6806","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6807","nonce":1437422725}]},"public_addr":"10.20.193.60:6803/1437422725","cluster_addr":"10.20.193.60:6805/1437422725","heartbeat_back_addr":"10.20.193.60:6809/1437422725","heartbeat_front_addr":"10.20.193.60:6807/1437422725","state":["exists","up"]},{"osd":1,"uuid":"6ebac1ed-ac24-4619-94c0-c4cfe9c29d08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6818","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6819","nonce":3794669997}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6820","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6821","nonce":3794669997}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6824","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6825","nonce":3794669997}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6822","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6823","nonce":3794669997}]},"public_addr":"10.20.193.60:6819/3794669997","cluster_addr":"10.20.193.60:6821/3794669997","heartbeat_back_addr":"10.20.193.60:6825/3794669997","heartbeat_front_addr":"10.20.193.60:6823/3794669997","state":["exists","up"]},{"osd":2,"uuid":"30a74eef-60f3-466e-8be5-b6e89425fb11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6810","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6811","nonce":4208307478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6812","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6813","nonce":4208307478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6816","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6817","nonce":4208307478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6814","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6815","nonce":4208307478}]},"public_addr":"10.20.193.60:6811/4208307478","cluster_addr":"10.20.193.60:6813/4208307478","heartbeat_back_addr":"10.20.193.60:6817/4208307478","heartbeat_front_addr":"10.20.193.60:6815/4208307478","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.840981+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.942404+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.876837+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:56:21.463 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-22T22:56:21.464 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-22T22:56:21.465 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-22T22:56:21.553 INFO:teuthology.orchestra.run.trial060.stdout:38654705668 2026-02-22T22:56:21.553 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:56:21.554 INFO:teuthology.orchestra.run.trial060.stdout:34359738373 2026-02-22T22:56:21.555 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:56:21.556 INFO:teuthology.orchestra.run.trial060.stdout:38654705667 2026-02-22T22:56:21.556 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:56:21.749 INFO:teuthology.orchestra.run.trial060.stdout:38654705667 2026-02-22T22:56:21.757 INFO:teuthology.orchestra.run.trial060.stdout:34359738372 2026-02-22T22:56:21.758 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705667 for osd.2 2026-02-22T22:56:21.759 INFO:teuthology.orchestra.run.trial060.stdout:38654705666 2026-02-22T22:56:21.767 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738372 for osd.0 2026-02-22T22:56:21.769 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705667 got 38654705666 for osd.1 2026-02-22T22:56:22.759 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:56:22.767 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:56:22.770 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:56:22.922 INFO:teuthology.orchestra.run.trial060.stdout:38654705667 2026-02-22T22:56:22.932 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705667 for osd.2 2026-02-22T22:56:22.960 INFO:teuthology.orchestra.run.trial060.stdout:34359738372 2026-02-22T22:56:22.961 INFO:teuthology.orchestra.run.trial060.stdout:38654705666 2026-02-22T22:56:22.969 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738372 for osd.0 2026-02-22T22:56:22.971 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705667 got 38654705666 for osd.1 2026-02-22T22:56:23.933 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:56:23.971 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:56:23.971 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:56:24.095 INFO:teuthology.orchestra.run.trial060.stdout:38654705668 2026-02-22T22:56:24.105 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705668 for osd.2 2026-02-22T22:56:24.106 DEBUG:teuthology.parallel:result is None 2026-02-22T22:56:24.136 INFO:teuthology.orchestra.run.trial060.stdout:38654705667 2026-02-22T22:56:24.139 INFO:teuthology.orchestra.run.trial060.stdout:34359738374 2026-02-22T22:56:24.146 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705667 got 38654705667 for osd.1 2026-02-22T22:56:24.146 DEBUG:teuthology.parallel:result is None 2026-02-22T22:56:24.149 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738374 for osd.0 2026-02-22T22:56:24.150 DEBUG:teuthology.parallel:result is None 2026-02-22T22:56:24.150 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T22:56:24.150 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:56:24.344 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:24.345 INFO:teuthology.orchestra.run.trial060.stderr:dumped all 2026-02-22T22:56:24.352 INFO:teuthology.orchestra.run.trial060.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T22:56:23.215990+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":10,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82476,"kb_used_data":1568,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033044,"statfs":{"total":289910292480,"available":289825837056,"internally_reserved":0,"allocated":1605632,"data_stored":1299359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":82767188},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.286739"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.934312+0000","last_change":"2026-02-22T22:56:20.934556+0000","last_active":"2026-02-22T22:56:20.934312+0000","last_peered":"2026-02-22T22:56:20.934312+0000","last_clean":"2026-02-22T22:56:20.934312+0000","last_became_active":"2026-02-22T22:56:18.932408+0000","last_became_peered":"2026-02-22T22:56:18.932408+0000","last_unstale":"2026-02-22T22:56:20.934312+0000","last_undegraded":"2026-02-22T22:56:20.934312+0000","last_fullsized":"2026-02-22T22:56:20.934312+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:45:37.059532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052600900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.934577+0000","last_change":"2026-02-22T22:56:20.934743+0000","last_active":"2026-02-22T22:56:20.934577+0000","last_peered":"2026-02-22T22:56:20.934577+0000","last_clean":"2026-02-22T22:56:20.934577+0000","last_became_active":"2026-02-22T22:56:18.931993+0000","last_became_peered":"2026-02-22T22:56:18.931993+0000","last_unstale":"2026-02-22T22:56:20.934577+0000","last_undegraded":"2026-02-22T22:56:20.934577+0000","last_fullsized":"2026-02-22T22:56:20.934577+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:55.729675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071762499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.934162+0000","last_change":"2026-02-22T22:56:20.934280+0000","last_active":"2026-02-22T22:56:20.934162+0000","last_peered":"2026-02-22T22:56:20.934162+0000","last_clean":"2026-02-22T22:56:20.934162+0000","last_became_active":"2026-02-22T22:56:18.932505+0000","last_became_peered":"2026-02-22T22:56:18.932505+0000","last_unstale":"2026-02-22T22:56:20.934162+0000","last_undegraded":"2026-02-22T22:56:20.934162+0000","last_fullsized":"2026-02-22T22:56:20.934162+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:50:12.129550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000270864,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.934291+0000","last_change":"2026-02-22T22:56:20.934550+0000","last_active":"2026-02-22T22:56:20.934291+0000","last_peered":"2026-02-22T22:56:20.934291+0000","last_clean":"2026-02-22T22:56:20.934291+0000","last_became_active":"2026-02-22T22:56:18.932003+0000","last_became_peered":"2026-02-22T22:56:18.932003+0000","last_unstale":"2026-02-22T22:56:20.934291+0000","last_undegraded":"2026-02-22T22:56:20.934291+0000","last_fullsized":"2026-02-22T22:56:20.934291+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:26:14.433190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054626700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.933457+0000","last_change":"2026-02-22T22:56:20.932755+0000","last_active":"2026-02-22T22:56:20.933457+0000","last_peered":"2026-02-22T22:56:20.933457+0000","last_clean":"2026-02-22T22:56:20.933457+0000","last_became_active":"2026-02-22T22:56:18.932278+0000","last_became_peered":"2026-02-22T22:56:18.932278+0000","last_unstale":"2026-02-22T22:56:20.933457+0000","last_undegraded":"2026-02-22T22:56:20.933457+0000","last_fullsized":"2026-02-22T22:56:20.933457+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:54:51.140871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066678099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:21.005975+0000","last_change":"2026-02-22T22:56:21.006101+0000","last_active":"2026-02-22T22:56:21.005975+0000","last_peered":"2026-02-22T22:56:21.005975+0000","last_clean":"2026-02-22T22:56:21.005975+0000","last_became_active":"2026-02-22T22:56:18.932279+0000","last_became_peered":"2026-02-22T22:56:18.932279+0000","last_unstale":"2026-02-22T22:56:21.005975+0000","last_undegraded":"2026-02-22T22:56:21.005975+0000","last_fullsized":"2026-02-22T22:56:21.005975+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:12:56.667350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023402500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:21.006016+0000","last_change":"2026-02-22T22:56:21.006207+0000","last_active":"2026-02-22T22:56:21.006016+0000","last_peered":"2026-02-22T22:56:21.006016+0000","last_clean":"2026-02-22T22:56:21.006016+0000","last_became_active":"2026-02-22T22:56:18.931984+0000","last_became_peered":"2026-02-22T22:56:18.931984+0000","last_unstale":"2026-02-22T22:56:21.006016+0000","last_undegraded":"2026-02-22T22:56:21.006016+0000","last_fullsized":"2026-02-22T22:56:21.006016+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:36.205745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033533400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.934145+0000","last_change":"2026-02-22T22:56:20.934279+0000","last_active":"2026-02-22T22:56:20.934145+0000","last_peered":"2026-02-22T22:56:20.934145+0000","last_clean":"2026-02-22T22:56:20.934145+0000","last_became_active":"2026-02-22T22:56:18.931925+0000","last_became_peered":"2026-02-22T22:56:18.931925+0000","last_unstale":"2026-02-22T22:56:20.934145+0000","last_undegraded":"2026-02-22T22:56:20.934145+0000","last_fullsized":"2026-02-22T22:56:20.934145+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:28:58.350535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000296552,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.933844+0000","last_change":"2026-02-22T22:56:14.922932+0000","last_active":"2026-02-22T22:56:20.933844+0000","last_peered":"2026-02-22T22:56:20.933844+0000","last_clean":"2026-02-22T22:56:20.933844+0000","last_became_active":"2026-02-22T22:56:14.921193+0000","last_became_peered":"2026-02-22T22:56:14.921193+0000","last_unstale":"2026-02-22T22:56:20.933844+0000","last_undegraded":"2026-02-22T22:56:20.933844+0000","last_fullsized":"2026-02-22T22:56:20.933844+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_clean_scrub_stamp":"2026-02-22T22:56:13.916808+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:23:02.729111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27096,"kb_used_data":128,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344744,"statfs":{"total":96636764160,"available":96609017856,"internally_reserved":0,"allocated":131072,"data_stored":37155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27589063},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705667,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":631102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27589063},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738374,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27692,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344148,"statfs":{"total":96636764160,"available":96608407552,"internally_reserved":0,"allocated":737280,"data_stored":631102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:56:24.353 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:56:24.510 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:24.511 INFO:teuthology.orchestra.run.trial060.stderr:dumped all 2026-02-22T22:56:24.518 INFO:teuthology.orchestra.run.trial060.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T22:56:23.215990+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":10,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82476,"kb_used_data":1568,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033044,"statfs":{"total":289910292480,"available":289825837056,"internally_reserved":0,"allocated":1605632,"data_stored":1299359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":82767188},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.286739"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.934312+0000","last_change":"2026-02-22T22:56:20.934556+0000","last_active":"2026-02-22T22:56:20.934312+0000","last_peered":"2026-02-22T22:56:20.934312+0000","last_clean":"2026-02-22T22:56:20.934312+0000","last_became_active":"2026-02-22T22:56:18.932408+0000","last_became_peered":"2026-02-22T22:56:18.932408+0000","last_unstale":"2026-02-22T22:56:20.934312+0000","last_undegraded":"2026-02-22T22:56:20.934312+0000","last_fullsized":"2026-02-22T22:56:20.934312+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:45:37.059532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052600900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.934577+0000","last_change":"2026-02-22T22:56:20.934743+0000","last_active":"2026-02-22T22:56:20.934577+0000","last_peered":"2026-02-22T22:56:20.934577+0000","last_clean":"2026-02-22T22:56:20.934577+0000","last_became_active":"2026-02-22T22:56:18.931993+0000","last_became_peered":"2026-02-22T22:56:18.931993+0000","last_unstale":"2026-02-22T22:56:20.934577+0000","last_undegraded":"2026-02-22T22:56:20.934577+0000","last_fullsized":"2026-02-22T22:56:20.934577+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:55.729675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071762499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.934162+0000","last_change":"2026-02-22T22:56:20.934280+0000","last_active":"2026-02-22T22:56:20.934162+0000","last_peered":"2026-02-22T22:56:20.934162+0000","last_clean":"2026-02-22T22:56:20.934162+0000","last_became_active":"2026-02-22T22:56:18.932505+0000","last_became_peered":"2026-02-22T22:56:18.932505+0000","last_unstale":"2026-02-22T22:56:20.934162+0000","last_undegraded":"2026-02-22T22:56:20.934162+0000","last_fullsized":"2026-02-22T22:56:20.934162+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:50:12.129550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000270864,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.934291+0000","last_change":"2026-02-22T22:56:20.934550+0000","last_active":"2026-02-22T22:56:20.934291+0000","last_peered":"2026-02-22T22:56:20.934291+0000","last_clean":"2026-02-22T22:56:20.934291+0000","last_became_active":"2026-02-22T22:56:18.932003+0000","last_became_peered":"2026-02-22T22:56:18.932003+0000","last_unstale":"2026-02-22T22:56:20.934291+0000","last_undegraded":"2026-02-22T22:56:20.934291+0000","last_fullsized":"2026-02-22T22:56:20.934291+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:26:14.433190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054626700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.933457+0000","last_change":"2026-02-22T22:56:20.932755+0000","last_active":"2026-02-22T22:56:20.933457+0000","last_peered":"2026-02-22T22:56:20.933457+0000","last_clean":"2026-02-22T22:56:20.933457+0000","last_became_active":"2026-02-22T22:56:18.932278+0000","last_became_peered":"2026-02-22T22:56:18.932278+0000","last_unstale":"2026-02-22T22:56:20.933457+0000","last_undegraded":"2026-02-22T22:56:20.933457+0000","last_fullsized":"2026-02-22T22:56:20.933457+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:54:51.140871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066678099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:21.005975+0000","last_change":"2026-02-22T22:56:21.006101+0000","last_active":"2026-02-22T22:56:21.005975+0000","last_peered":"2026-02-22T22:56:21.005975+0000","last_clean":"2026-02-22T22:56:21.005975+0000","last_became_active":"2026-02-22T22:56:18.932279+0000","last_became_peered":"2026-02-22T22:56:18.932279+0000","last_unstale":"2026-02-22T22:56:21.005975+0000","last_undegraded":"2026-02-22T22:56:21.005975+0000","last_fullsized":"2026-02-22T22:56:21.005975+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:12:56.667350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023402500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:21.006016+0000","last_change":"2026-02-22T22:56:21.006207+0000","last_active":"2026-02-22T22:56:21.006016+0000","last_peered":"2026-02-22T22:56:21.006016+0000","last_clean":"2026-02-22T22:56:21.006016+0000","last_became_active":"2026-02-22T22:56:18.931984+0000","last_became_peered":"2026-02-22T22:56:18.931984+0000","last_unstale":"2026-02-22T22:56:21.006016+0000","last_undegraded":"2026-02-22T22:56:21.006016+0000","last_fullsized":"2026-02-22T22:56:21.006016+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:36.205745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033533400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.934145+0000","last_change":"2026-02-22T22:56:20.934279+0000","last_active":"2026-02-22T22:56:20.934145+0000","last_peered":"2026-02-22T22:56:20.934145+0000","last_clean":"2026-02-22T22:56:20.934145+0000","last_became_active":"2026-02-22T22:56:18.931925+0000","last_became_peered":"2026-02-22T22:56:18.931925+0000","last_unstale":"2026-02-22T22:56:20.934145+0000","last_undegraded":"2026-02-22T22:56:20.934145+0000","last_fullsized":"2026-02-22T22:56:20.934145+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:28:58.350535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000296552,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:56:20.933844+0000","last_change":"2026-02-22T22:56:14.922932+0000","last_active":"2026-02-22T22:56:20.933844+0000","last_peered":"2026-02-22T22:56:20.933844+0000","last_clean":"2026-02-22T22:56:20.933844+0000","last_became_active":"2026-02-22T22:56:14.921193+0000","last_became_peered":"2026-02-22T22:56:14.921193+0000","last_unstale":"2026-02-22T22:56:20.933844+0000","last_undegraded":"2026-02-22T22:56:20.933844+0000","last_fullsized":"2026-02-22T22:56:20.933844+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_clean_scrub_stamp":"2026-02-22T22:56:13.916808+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:23:02.729111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27096,"kb_used_data":128,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344744,"statfs":{"total":96636764160,"available":96609017856,"internally_reserved":0,"allocated":131072,"data_stored":37155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27589063},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705667,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":631102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27589063},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738374,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27692,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344148,"statfs":{"total":96636764160,"available":96608407552,"internally_reserved":0,"allocated":737280,"data_stored":631102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:56:24.519 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T22:56:24.520 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-22T22:56:24.520 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-22T22:56:24.520 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-22T22:56:24.691 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:24.691 INFO:teuthology.orchestra.run.trial060.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T22:56:24.699 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-22T22:56:24.699 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T22:56:24.711 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-22T22:56:24.711 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T22:56:24.711 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-bitmap supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-exclusive-lock}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64284', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial060.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYMkwFUQ6cXE+ZjCzk3UVcU7g01xRvGge6562s1gf58Hws4h6Io7AwM+0Sbn9MFfRklivaxmFE6mipfm0hckrE=', 'trial082.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPglXcHDxID8MMv+LLgiKjwWZtASzV0uUItaQbBbwe7w6JE63GHaCY+64iPiIcJLBXEE5mo+m1+LkfyxusecEPE='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock', 'RBD_MIRROR_CONFIG_KEY': '1', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T22:56:24.712 DEBUG:teuthology.orchestra.run.trial060:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T22:56:24.714 DEBUG:teuthology.orchestra.run.trial082:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T22:56:24.731 DEBUG:teuthology.orchestra.run.trial060:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:56:24.771 DEBUG:teuthology.orchestra.run.trial082:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:56:24.798 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:24.798 DEBUG:teuthology.orchestra.run.trial082:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T22:56:24.855 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T22:56:24.856 DEBUG:teuthology.orchestra.run.trial082:> stat /dev/vg_nvme/lv_1 2026-02-22T22:56:24.914 INFO:teuthology.orchestra.run.trial082.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T22:56:24.914 INFO:teuthology.orchestra.run.trial082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:56:24.914 INFO:teuthology.orchestra.run.trial082.stdout:Device: 6h/6d Inode: 1308 Links: 1 2026-02-22T22:56:24.914 INFO:teuthology.orchestra.run.trial082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:56:24.914 INFO:teuthology.orchestra.run.trial082.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:56:24.914 INFO:teuthology.orchestra.run.trial082.stdout:Access: 2026-02-22 22:55:40.911646401 +0000 2026-02-22T22:56:24.914 INFO:teuthology.orchestra.run.trial082.stdout:Modify: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:24.914 INFO:teuthology.orchestra.run.trial082.stdout:Change: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:24.915 INFO:teuthology.orchestra.run.trial082.stdout: Birth: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:24.915 DEBUG:teuthology.orchestra.run.trial082:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T22:56:24.977 INFO:teuthology.orchestra.run.trial082.stderr:1+0 records in 2026-02-22T22:56:24.978 INFO:teuthology.orchestra.run.trial082.stderr:1+0 records out 2026-02-22T22:56:24.978 INFO:teuthology.orchestra.run.trial082.stderr:512 bytes copied, 0.00014564 s, 3.5 MB/s 2026-02-22T22:56:24.979 DEBUG:teuthology.orchestra.run.trial082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T22:56:25.035 DEBUG:teuthology.orchestra.run.trial082:> stat /dev/vg_nvme/lv_2 2026-02-22T22:56:25.093 INFO:teuthology.orchestra.run.trial082.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T22:56:25.094 INFO:teuthology.orchestra.run.trial082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:56:25.094 INFO:teuthology.orchestra.run.trial082.stdout:Device: 6h/6d Inode: 1303 Links: 1 2026-02-22T22:56:25.094 INFO:teuthology.orchestra.run.trial082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:56:25.094 INFO:teuthology.orchestra.run.trial082.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:56:25.094 INFO:teuthology.orchestra.run.trial082.stdout:Access: 2026-02-22 22:55:40.911646401 +0000 2026-02-22T22:56:25.094 INFO:teuthology.orchestra.run.trial082.stdout:Modify: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:25.094 INFO:teuthology.orchestra.run.trial082.stdout:Change: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:25.094 INFO:teuthology.orchestra.run.trial082.stdout: Birth: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:25.095 DEBUG:teuthology.orchestra.run.trial082:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T22:56:25.156 INFO:teuthology.orchestra.run.trial082.stderr:1+0 records in 2026-02-22T22:56:25.156 INFO:teuthology.orchestra.run.trial082.stderr:1+0 records out 2026-02-22T22:56:25.157 INFO:teuthology.orchestra.run.trial082.stderr:512 bytes copied, 0.000144639 s, 3.5 MB/s 2026-02-22T22:56:25.158 DEBUG:teuthology.orchestra.run.trial082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T22:56:25.216 DEBUG:teuthology.orchestra.run.trial082:> stat /dev/vg_nvme/lv_3 2026-02-22T22:56:25.273 INFO:teuthology.orchestra.run.trial082.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T22:56:25.273 INFO:teuthology.orchestra.run.trial082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:56:25.273 INFO:teuthology.orchestra.run.trial082.stdout:Device: 6h/6d Inode: 1294 Links: 1 2026-02-22T22:56:25.273 INFO:teuthology.orchestra.run.trial082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:56:25.273 INFO:teuthology.orchestra.run.trial082.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:56:25.274 INFO:teuthology.orchestra.run.trial082.stdout:Access: 2026-02-22 22:55:40.911646401 +0000 2026-02-22T22:56:25.274 INFO:teuthology.orchestra.run.trial082.stdout:Modify: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:25.274 INFO:teuthology.orchestra.run.trial082.stdout:Change: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:25.274 INFO:teuthology.orchestra.run.trial082.stdout: Birth: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:25.274 DEBUG:teuthology.orchestra.run.trial082:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T22:56:25.337 INFO:teuthology.orchestra.run.trial082.stderr:1+0 records in 2026-02-22T22:56:25.337 INFO:teuthology.orchestra.run.trial082.stderr:1+0 records out 2026-02-22T22:56:25.337 INFO:teuthology.orchestra.run.trial082.stderr:512 bytes copied, 0.000159977 s, 3.2 MB/s 2026-02-22T22:56:25.338 DEBUG:teuthology.orchestra.run.trial082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T22:56:25.396 DEBUG:teuthology.orchestra.run.trial082:> stat /dev/vg_nvme/lv_4 2026-02-22T22:56:25.454 INFO:teuthology.orchestra.run.trial082.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T22:56:25.454 INFO:teuthology.orchestra.run.trial082.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:56:25.454 INFO:teuthology.orchestra.run.trial082.stdout:Device: 6h/6d Inode: 1297 Links: 1 2026-02-22T22:56:25.454 INFO:teuthology.orchestra.run.trial082.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:56:25.454 INFO:teuthology.orchestra.run.trial082.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:56:25.454 INFO:teuthology.orchestra.run.trial082.stdout:Access: 2026-02-22 22:55:40.911646401 +0000 2026-02-22T22:56:25.454 INFO:teuthology.orchestra.run.trial082.stdout:Modify: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:25.454 INFO:teuthology.orchestra.run.trial082.stdout:Change: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:25.455 INFO:teuthology.orchestra.run.trial082.stdout: Birth: 2026-02-22 22:54:47.959218856 +0000 2026-02-22T22:56:25.455 DEBUG:teuthology.orchestra.run.trial082:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T22:56:25.517 INFO:teuthology.orchestra.run.trial082.stderr:1+0 records in 2026-02-22T22:56:25.517 INFO:teuthology.orchestra.run.trial082.stderr:1+0 records out 2026-02-22T22:56:25.517 INFO:teuthology.orchestra.run.trial082.stderr:512 bytes copied, 0.00015072 s, 3.4 MB/s 2026-02-22T22:56:25.518 DEBUG:teuthology.orchestra.run.trial082:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T22:56:25.577 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:56:25.578 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial082.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:56:25.578 INFO:tasks.ceph:Generating config... 2026-02-22T22:56:25.580 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T22:56:25.580 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T22:56:25.580 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T22:56:25.580 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T22:56:25.580 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:25.580 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:25.580 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:25.581 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:25.582 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T22:56:25.582 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T22:56:25.582 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T22:56:25.582 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T22:56:25.582 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T22:56:25.582 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T22:56:25.582 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T22:56:25.582 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T22:56:25.582 INFO:tasks.ceph:[osd] bdev async discard = True 2026-02-22T22:56:25.582 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-02-22T22:56:25.582 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2026-02-22T22:56:25.582 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T22:56:25.582 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T22:56:25.583 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T22:56:25.583 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T22:56:25.583 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T22:56:25.583 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T22:56:25.583 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T22:56:25.583 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T22:56:25.583 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T22:56:25.583 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T22:56:25.583 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T22:56:25.583 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T22:56:25.583 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-22T22:56:25.584 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-22T22:56:25.651 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster2.keyring 2026-02-22T22:56:25.654 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-22T22:56:25.732 DEBUG:teuthology.orchestra.run.trial082:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:56:25.795 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.82')] 2026-02-22T22:56:25.795 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.82', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T22:56:25.795 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:25.796 DEBUG:teuthology.orchestra.run.trial082:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:56:25.850 DEBUG:teuthology.orchestra.run.trial082:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.82 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:56:25.924 INFO:teuthology.orchestra.run.trial082.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:56:25.924 INFO:teuthology.orchestra.run.trial082.stdout:monmaptool: generated fsid 159baabb-b8a7-4828-b518-eb266dc5b284 2026-02-22T22:56:25.924 INFO:teuthology.orchestra.run.trial082.stdout:setting min_mon_release = quincy 2026-02-22T22:56:25.924 INFO:teuthology.orchestra.run.trial082.stdout:epoch 0 2026-02-22T22:56:25.924 INFO:teuthology.orchestra.run.trial082.stdout:fsid 159baabb-b8a7-4828-b518-eb266dc5b284 2026-02-22T22:56:25.924 INFO:teuthology.orchestra.run.trial082.stdout:last_changed 2026-02-22T22:56:25.922708+0000 2026-02-22T22:56:25.924 INFO:teuthology.orchestra.run.trial082.stdout:created 2026-02-22T22:56:25.922708+0000 2026-02-22T22:56:25.924 INFO:teuthology.orchestra.run.trial082.stdout:min_mon_release 17 (quincy) 2026-02-22T22:56:25.925 INFO:teuthology.orchestra.run.trial082.stdout:election_strategy: 1 2026-02-22T22:56:25.925 INFO:teuthology.orchestra.run.trial082.stdout:0: [v2:10.20.193.82:3300/0,v1:10.20.193.82:6789/0] mon.a 2026-02-22T22:56:25.925 INFO:teuthology.orchestra.run.trial082.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-22T22:56:25.926 DEBUG:teuthology.orchestra.run.trial082:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:56:25.983 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 159baabb-b8a7-4828-b518-eb266dc5b284... 2026-02-22T22:56:25.985 DEBUG:teuthology.orchestra.run.trial060:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T22:56:25.988 DEBUG:teuthology.orchestra.run.trial082:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T22:56:26.027 INFO:teuthology.orchestra.run.trial060.stdout:[global] 2026-02-22T22:56:26.027 INFO:teuthology.orchestra.run.trial060.stdout: chdir = "" 2026-02-22T22:56:26.027 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:56:26.027 INFO:teuthology.orchestra.run.trial060.stdout: auth supported = cephx 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: filestore xattr use omap = true 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: mon clock drift allowed = 1.000 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: osd crush chooseleaf type = 0 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: auth debug = true 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: ms die on old message = true 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: ms die on bug = true 2026-02-22T22:56:26.028 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: mon pg warn max object skew = 0 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: osd pool default size = 2 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: mon osd allow primary affinity = true 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: mon osd allow pg remap = true 2026-02-22T22:56:26.029 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on no sortbitwise = false 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on too few osds = false 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: mon_allow_pool_size_one = true 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: osd default data pool replay window = 5 2026-02-22T22:56:26.030 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: mon allow pool delete = true 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: mon cluster log file level = debug 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: debug asserts on shutdown = true 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: mon health detail to clog = false 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: mon host = 10.20.193.82 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: mon client directed command retry = 5 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: mon warn on pool no app = False 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: ms inject socket failures = 5000 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: fsid = 159baabb-b8a7-4828-b518-eb266dc5b284 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout:[osd] 2026-02-22T22:56:26.031 INFO:teuthology.orchestra.run.trial060.stdout: osd journal size = 100 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: osd scrub load threshold = 5.0 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: osd scrub max interval = 600 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: osd mclock skip benchmark = true 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: osd recover clone overlap = true 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: osd recovery max chunk = 1048576 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: osd debug shutdown = true 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: osd debug op order = true 2026-02-22T22:56:26.032 INFO:teuthology.orchestra.run.trial060.stdout: osd debug verify stray on activate = true 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: osd debug trim objects = true 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: osd open classes on start = true 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: osd debug pg log writeout = true 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: osd map max advance = 10 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: journal zero on create = true 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.033 INFO:teuthology.orchestra.run.trial060.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: filestore apply finisher threads = 3 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: bdev debug aio = true 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: osd debug misdirected ops = true 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: bdev async discard = True 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: bdev enable discard = True 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: bluestore allocator = bitmap 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: bluestore block size = 96636764160 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: bluestore fsck on mount = True 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: debug bluefs = 1/20 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: debug bluestore = 1/20 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: debug ms = 1 2026-02-22T22:56:26.034 INFO:teuthology.orchestra.run.trial060.stdout: debug osd = 20 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: debug rocksdb = 4/10 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: mon osd full ratio = 0.9 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: osd objectstore = bluestore 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout:[mgr] 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: debug ms = 1 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: debug mgr = 20 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: debug mon = 20 2026-02-22T22:56:26.035 INFO:teuthology.orchestra.run.trial060.stdout: debug auth = 20 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: mgr/telemetry/nag = false 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout:[mon] 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: debug ms = 1 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: debug mon = 20 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: debug paxos = 20 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: debug auth = 20 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: mon data avail warn = 5 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: mon mgr mkfs grace = 240 2026-02-22T22:56:26.036 INFO:teuthology.orchestra.run.trial060.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: mon osd reporter subtree level = osd 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: mon osd prime pg temp = true 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.037 INFO:teuthology.orchestra.run.trial060.stdout: # 1m isn't quite enough 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout:[client] 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: rgw cache enabled = true 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: rgw enable ops log = true 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: rgw enable usage log = true 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: debug rbd = 20 2026-02-22T22:56:26.038 INFO:teuthology.orchestra.run.trial060.stdout: debug rbd_mirror = 15 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout: log to stderr = False 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout: rbd default features = 125 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout:[mon.a] 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.0] 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.1] 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.2] 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.039 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.3] 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.4] 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.5] 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout:[client.mirror.6] 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.040 INFO:teuthology.orchestra.run.trial060.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.064 INFO:teuthology.orchestra.run.trial082.stdout:[global] 2026-02-22T22:56:26.064 INFO:teuthology.orchestra.run.trial082.stdout: chdir = "" 2026-02-22T22:56:26.064 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:56:26.064 INFO:teuthology.orchestra.run.trial082.stdout: auth supported = cephx 2026-02-22T22:56:26.064 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.064 INFO:teuthology.orchestra.run.trial082.stdout: filestore xattr use omap = true 2026-02-22T22:56:26.064 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: mon clock drift allowed = 1.000 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: osd crush chooseleaf type = 0 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: auth debug = true 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: ms die on old message = true 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: ms die on bug = true 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: mon pg warn max object skew = 0 2026-02-22T22:56:26.065 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: osd pool default size = 2 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: mon osd allow primary affinity = true 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: mon osd allow pg remap = true 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on no sortbitwise = false 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:56:26.066 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on too few osds = false 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: mon_allow_pool_size_one = true 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: osd default data pool replay window = 5 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: mon allow pool delete = true 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: mon cluster log file level = debug 2026-02-22T22:56:26.067 INFO:teuthology.orchestra.run.trial082.stdout: debug asserts on shutdown = true 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: mon health detail to clog = false 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: mon host = 10.20.193.82 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: mon client directed command retry = 5 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: mon warn on pool no app = False 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: ms inject socket failures = 5000 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: fsid = 159baabb-b8a7-4828-b518-eb266dc5b284 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout:[osd] 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: osd journal size = 100 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: osd scrub load threshold = 5.0 2026-02-22T22:56:26.068 INFO:teuthology.orchestra.run.trial082.stdout: osd scrub max interval = 600 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: osd mclock skip benchmark = true 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: osd recover clone overlap = true 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: osd recovery max chunk = 1048576 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: osd debug shutdown = true 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: osd debug op order = true 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: osd debug verify stray on activate = true 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: osd debug trim objects = true 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: osd open classes on start = true 2026-02-22T22:56:26.069 INFO:teuthology.orchestra.run.trial082.stdout: osd debug pg log writeout = true 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: osd map max advance = 10 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: journal zero on create = true 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: filestore apply finisher threads = 3 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: bdev debug aio = true 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: osd debug misdirected ops = true 2026-02-22T22:56:26.070 INFO:teuthology.orchestra.run.trial082.stdout: bdev async discard = True 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: bdev enable discard = True 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: bluestore allocator = bitmap 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: bluestore block size = 96636764160 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: bluestore fsck on mount = True 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: debug bluefs = 1/20 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: debug bluestore = 1/20 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: debug ms = 1 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: debug osd = 20 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: debug rocksdb = 4/10 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: mon osd full ratio = 0.9 2026-02-22T22:56:26.071 INFO:teuthology.orchestra.run.trial082.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: osd objectstore = bluestore 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout:[mgr] 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: debug ms = 1 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: debug mgr = 20 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: debug mon = 20 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: debug auth = 20 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: mgr/telemetry/nag = false 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.072 INFO:teuthology.orchestra.run.trial082.stdout:[mon] 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: debug ms = 1 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: debug mon = 20 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: debug paxos = 20 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: debug auth = 20 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: mon data avail warn = 5 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: mon mgr mkfs grace = 240 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: mon osd reporter subtree level = osd 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: mon osd prime pg temp = true 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:56:26.073 INFO:teuthology.orchestra.run.trial082.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: # 1m isn't quite enough 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:26.074 INFO:teuthology.orchestra.run.trial082.stdout:[client] 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout: rgw cache enabled = true 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout: rgw enable ops log = true 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout: rgw enable usage log = true 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout: debug rbd = 20 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout: debug rbd_mirror = 15 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout: log to stderr = False 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout: rbd default features = 125 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout:[mon.a] 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.0] 2026-02-22T22:56:26.075 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.1] 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.2] 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.3] 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.4] 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.076 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.077 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.5] 2026-02-22T22:56:26.077 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.077 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.077 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.6] 2026-02-22T22:56:26.077 INFO:teuthology.orchestra.run.trial082.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:56:26.077 INFO:teuthology.orchestra.run.trial082.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:56:26.078 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-22T22:56:26.078 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-22T22:56:26.162 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T22:56:26.162 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:26.163 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-22T22:56:26.222 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:26.222 DEBUG:teuthology.orchestra.run.trial082:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-22T22:56:26.280 INFO:tasks.ceph:Sending monmap to node ubuntu@trial060.front.sepia.ceph.com 2026-02-22T22:56:26.280 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:26.280 DEBUG:teuthology.orchestra.run.trial060:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T22:56:26.280 DEBUG:teuthology.orchestra.run.trial060:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:56:26.315 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:26.315 DEBUG:teuthology.orchestra.run.trial060:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:56:26.371 INFO:tasks.ceph:Sending monmap to node ubuntu@trial082.front.sepia.ceph.com 2026-02-22T22:56:26.372 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:26.372 DEBUG:teuthology.orchestra.run.trial082:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T22:56:26.372 DEBUG:teuthology.orchestra.run.trial082:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:56:26.407 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:26.408 DEBUG:teuthology.orchestra.run.trial082:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:56:26.463 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T22:56:26.463 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T22:56:26.464 DEBUG:teuthology.orchestra.run.trial060:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T22:56:26.506 INFO:teuthology.orchestra.run.trial060.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T22:56:26.509 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T22:56:26.509 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T22:56:26.509 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-22T22:56:26.582 INFO:teuthology.orchestra.run.trial060.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-22T22:56:26.593 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T22:56:26.628 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T22:56:26.639 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T22:56:26.712 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T22:56:26.723 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T22:56:26.800 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T22:56:26.811 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T22:56:26.886 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T22:56:26.897 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T22:56:26.971 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T22:56:26.982 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T22:56:27.066 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T22:56:27.078 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T22:56:27.153 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T22:56:27.164 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T22:56:27.239 INFO:teuthology.orchestra.run.trial082.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T22:56:27.249 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T22:56:27.250 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial060.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial082.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:56:27.250 DEBUG:teuthology.orchestra.run.trial082:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-22T22:56:27.314 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:56:27.314 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-22T22:56:27.314 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial082.front.sepia.ceph.com 2026-02-22T22:56:27.314 DEBUG:teuthology.orchestra.run.trial082:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T22:56:27.379 INFO:teuthology.orchestra.run.trial082.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:56:27.379 INFO:teuthology.orchestra.run.trial082.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:56:27.379 INFO:teuthology.orchestra.run.trial082.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:56:27.379 INFO:teuthology.orchestra.run.trial082.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:56:27.379 INFO:teuthology.orchestra.run.trial082.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:56:27.379 INFO:teuthology.orchestra.run.trial082.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:56:27.379 INFO:teuthology.orchestra.run.trial082.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:56:27.379 INFO:teuthology.orchestra.run.trial082.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:56:27.379 INFO:teuthology.orchestra.run.trial082.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:56:27.380 INFO:teuthology.orchestra.run.trial082.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:56:27.952 INFO:teuthology.orchestra.run.trial082.stdout:Discarding blocks...Done. 2026-02-22T22:56:27.954 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial082.front.sepia.ceph.com -o noatime 2026-02-22T22:56:27.955 DEBUG:teuthology.orchestra.run.trial082:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-22T22:56:28.023 DEBUG:teuthology.orchestra.run.trial082:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-22T22:56:28.090 DEBUG:teuthology.orchestra.run.trial082:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-22T22:56:28.154 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:56:28.154 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-22T22:56:28.154 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial082.front.sepia.ceph.com 2026-02-22T22:56:28.154 DEBUG:teuthology.orchestra.run.trial082:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T22:56:28.218 INFO:teuthology.orchestra.run.trial082.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:56:28.219 INFO:teuthology.orchestra.run.trial082.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:56:28.219 INFO:teuthology.orchestra.run.trial082.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:56:28.219 INFO:teuthology.orchestra.run.trial082.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:56:28.219 INFO:teuthology.orchestra.run.trial082.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:56:28.219 INFO:teuthology.orchestra.run.trial082.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:56:28.219 INFO:teuthology.orchestra.run.trial082.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:56:28.219 INFO:teuthology.orchestra.run.trial082.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:56:28.219 INFO:teuthology.orchestra.run.trial082.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:56:28.219 INFO:teuthology.orchestra.run.trial082.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:56:28.776 INFO:teuthology.orchestra.run.trial082.stdout:Discarding blocks...Done. 2026-02-22T22:56:28.778 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial082.front.sepia.ceph.com -o noatime 2026-02-22T22:56:28.778 DEBUG:teuthology.orchestra.run.trial082:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-22T22:56:28.845 DEBUG:teuthology.orchestra.run.trial082:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-22T22:56:28.914 DEBUG:teuthology.orchestra.run.trial082:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-22T22:56:28.978 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:56:28.978 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-22T22:56:28.978 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial082.front.sepia.ceph.com 2026-02-22T22:56:28.978 DEBUG:teuthology.orchestra.run.trial082:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T22:56:29.042 INFO:teuthology.orchestra.run.trial082.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:56:29.042 INFO:teuthology.orchestra.run.trial082.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:56:29.043 INFO:teuthology.orchestra.run.trial082.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:56:29.043 INFO:teuthology.orchestra.run.trial082.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:56:29.043 INFO:teuthology.orchestra.run.trial082.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:56:29.043 INFO:teuthology.orchestra.run.trial082.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:56:29.043 INFO:teuthology.orchestra.run.trial082.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:56:29.043 INFO:teuthology.orchestra.run.trial082.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:56:29.043 INFO:teuthology.orchestra.run.trial082.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:56:29.043 INFO:teuthology.orchestra.run.trial082.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:56:29.599 INFO:teuthology.orchestra.run.trial082.stdout:Discarding blocks...Done. 2026-02-22T22:56:29.602 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial082.front.sepia.ceph.com -o noatime 2026-02-22T22:56:29.602 DEBUG:teuthology.orchestra.run.trial082:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-22T22:56:29.669 DEBUG:teuthology.orchestra.run.trial082:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-22T22:56:29.736 DEBUG:teuthology.orchestra.run.trial082:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:56:29.814 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:29.812+0000 7f40b7013780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-22T22:56:29.814 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:29.812+0000 7f40b7013780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-22T22:56:29.814 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:29.812+0000 7f40b7013780 -1 bdev(0x55d83818f800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-22T22:56:29.814 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:29.812+0000 7f40b7013780 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-22T22:56:33.705 DEBUG:teuthology.orchestra.run.trial082:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-22T22:56:33.770 DEBUG:teuthology.orchestra.run.trial082:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:56:33.850 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:33.849+0000 7f39aec10780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-22T22:56:33.851 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:33.849+0000 7f39aec10780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-22T22:56:33.851 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:33.849+0000 7f39aec10780 -1 bdev(0x55be1dc69800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-22T22:56:33.851 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:33.849+0000 7f39aec10780 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-22T22:56:37.739 DEBUG:teuthology.orchestra.run.trial082:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-22T22:56:37.804 DEBUG:teuthology.orchestra.run.trial082:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:56:37.884 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:37.882+0000 7f1022e11780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-22T22:56:37.884 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:37.882+0000 7f1022e11780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-22T22:56:37.884 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:37.882+0000 7f1022e11780 -1 bdev(0x55e214f03800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-22T22:56:37.884 INFO:teuthology.orchestra.run.trial082.stderr:2026-02-22T22:56:37.882+0000 7f1022e11780 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-22T22:56:41.774 DEBUG:teuthology.orchestra.run.trial082:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-22T22:56:41.838 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T22:56:41.838 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:41.838 DEBUG:teuthology.orchestra.run.trial060:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-22T22:56:41.864 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:41.864 DEBUG:teuthology.orchestra.run.trial082:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-22T22:56:41.903 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:41.903 DEBUG:teuthology.orchestra.run.trial082:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-22T22:56:41.966 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:41.967 DEBUG:teuthology.orchestra.run.trial082:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-22T22:56:42.030 DEBUG:teuthology.orchestra.run.trial060:> set -ex 2026-02-22T22:56:42.030 DEBUG:teuthology.orchestra.run.trial060:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-22T22:56:42.046 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:42.046 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-22T22:56:42.085 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:42.085 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-22T22:56:42.141 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:42.141 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-22T22:56:42.196 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:42.196 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-22T22:56:42.252 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:42.252 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-22T22:56:42.306 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:42.306 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-22T22:56:42.363 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:42.363 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-22T22:56:42.418 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:42.418 DEBUG:teuthology.orchestra.run.trial082:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-22T22:56:42.474 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T22:56:42.474 DEBUG:teuthology.orchestra.run.trial082:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[mgr.x] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtpU2tHHhAA5upivvIiBlLt0k8/gdtEhQ== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[osd.0] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCdiZtp2TF1MBAAkQLuCXJVvPL32852jX8akg== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[osd.1] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQChiZtpSRKsMhAALU9DBrwhlH2k32qkhAF1qg== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[osd.2] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCliZtp6lCgNBAAGIAkoCHNOwqwymhFBd8fYA== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[client.0] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtpHArQIhAA2u0UJ2mY23nP8qEU9+/06g== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtp7HFoJRAAxSKsJ9KLDt+ukcfRSAwwBA== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.0] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtpRq5oKhAAG0UU87I7ziy8MfMM9WTNaA== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.1] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtpzGaoLxAAewwQAXEYciQyIiUzVoe4mw== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.2] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtp4ZnMNBAAldjijeFiEcDO3BZvh8ZWOg== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.3] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtpSNrSORAA947XpKn4pjJ7ccD9j0MVzg== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.4] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCbiZtp84nmAxAAgArySjgCAQ034NH9rYvm/Q== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.5] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCbiZtpHVEVCRAASwKOV1V4bQm71tAPJt0Dbg== 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.6] 2026-02-22T22:56:42.536 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCbiZtpSKAwDhAAoIePBbyFIkIdOqDxr3oCvA== 2026-02-22T22:56:42.537 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T22:56:42.616 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:56:42.695 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:56:42.772 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:56:42.854 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:42.935 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:43.014 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:43.104 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:43.191 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:43.274 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:43.355 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:43.436 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:43.515 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:56:43.595 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T22:56:43.595 DEBUG:teuthology.orchestra.run.trial082:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-22T22:56:43.659 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-22T22:56:43.747 DEBUG:teuthology.orchestra.run.trial082:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-22T22:56:43.772 DEBUG:teuthology.orchestra.run.trial082:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:56:43.828 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-22T22:56:43.829 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-22T22:56:43.829 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-22T22:56:43.870 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-22T22:56:43.870 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-22T22:56:43.870 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-22T22:56:43.870 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-22T22:56:43.872 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-22T22:56:43.872 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T22:56:43.872 DEBUG:teuthology.orchestra.run.trial082:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-22T22:56:43.975 INFO:teuthology.orchestra.run.trial082.stderr:adjusted tunables profile to default 2026-02-22T22:56:43.984 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T22:56:43.984 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-22T22:56:43.984 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:43.984 DEBUG:teuthology.orchestra.run.trial082:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-22T22:56:44.009 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:44.009 DEBUG:teuthology.orchestra.run.trial082:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-22T22:56:44.075 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:56:44.075 DEBUG:teuthology.orchestra.run.trial082:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-22T22:56:44.138 DEBUG:teuthology.orchestra.run.trial082:> sudo ceph --cluster cluster2 osd new b46a050a-f95d-4c3b-aa52-8034685b24cb 0 2026-02-22T22:56:44.203 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.204+0000 7fd8c4ddf140 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T22:56:44.283 INFO:teuthology.orchestra.run.trial082.stdout:0 2026-02-22T22:56:44.289 DEBUG:teuthology.orchestra.run.trial082:> sudo ceph --cluster cluster2 osd new 7485287f-05fb-4da0-b9dc-4b04e779999d 1 2026-02-22T22:56:44.321 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.322+0000 7fd8c4ddf140 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T22:56:44.343 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.344+0000 7fd8c4ddf140 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T22:56:44.369 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.370+0000 7fd8c4ddf140 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T22:56:44.389 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.390+0000 7fd8c4ddf140 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T22:56:44.392 INFO:teuthology.orchestra.run.trial082.stdout:1 2026-02-22T22:56:44.400 DEBUG:teuthology.orchestra.run.trial082:> sudo ceph --cluster cluster2 osd new 953c1811-ccbc-4b30-9679-3c746f040954 2 2026-02-22T22:56:44.415 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.416+0000 7fd8c4ddf140 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T22:56:44.466 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.466+0000 7fd8c4ddf140 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T22:56:44.501 INFO:teuthology.orchestra.run.trial082.stdout:2 2026-02-22T22:56:44.509 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-22T22:56:44.509 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-22T22:56:44.511 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-22T22:56:44.511 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-22T22:56:44.511 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-22T22:56:44.513 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-22T22:56:44.513 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-22T22:56:44.513 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-22T22:56:44.515 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-22T22:56:44.515 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:56:44.524 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T22:56:44.524 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T22:56:44.524 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr: from numpy import show_config as show_numpy_config 2026-02-22T22:56:44.527 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.528+0000 7fd8c4ddf140 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T22:56:44.551 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.552+0000 7fd8c4ddf140 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T22:56:44.604 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.605+0000 7fd8c4ddf140 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2026-02-22T22:56:44.616 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:56:44.616 INFO:teuthology.orchestra.run.trial082.stdout:{"epoch":5,"fsid":"159baabb-b8a7-4828-b518-eb266dc5b284","created":"2026-02-22T22:56:43.930984+0000","modified":"2026-02-22T22:56:44.500294+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:56:44.500294+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"b46a050a-f95d-4c3b-aa52-8034685b24cb","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"7485287f-05fb-4da0-b9dc-4b04e779999d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"953c1811-ccbc-4b30-9679-3c746f040954","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:56:44.623 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-22T22:56:44.623 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T22:56:44.645 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.646+0000 7fd8c4ddf140 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T22:56:44.669 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.670+0000 7fd8c4ddf140 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T22:56:44.692 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.693+0000 7fd8c4ddf140 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T22:56:44.716 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.717+0000 7fd8c4ddf140 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T22:56:44.737 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.738+0000 7fd8c4ddf140 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T22:56:44.892 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.893+0000 7fd8c4ddf140 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2026-02-22T22:56:44.925 DEBUG:teuthology.orchestra.run.trial082:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:56:44.948 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.949+0000 7fd8c4ddf140 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T22:56:44.969 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.970+0000 7fd8c4ddf140 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T22:56:44.991 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:44.991+0000 7fd8c4ddf140 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T22:56:45.021 INFO:teuthology.misc.health.trial082.stdout: 2026-02-22T22:56:45.021 INFO:teuthology.misc.health.trial082.stdout:{"epoch":5,"fsid":"159baabb-b8a7-4828-b518-eb266dc5b284","created":"2026-02-22T22:56:43.930984+0000","modified":"2026-02-22T22:56:44.500294+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:56:44.500294+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"b46a050a-f95d-4c3b-aa52-8034685b24cb","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"7485287f-05fb-4da0-b9dc-4b04e779999d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"953c1811-ccbc-4b30-9679-3c746f040954","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:56:45.028 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:56:45.043 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:45.044+0000 7fd8c4ddf140 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:56:45.064 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:45.064+0000 7fd8c4ddf140 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T22:56:45.115 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:45.116+0000 7fd8c4ddf140 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:56:45.436 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:45.436+0000 7fd8c4ddf140 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T22:56:45.456 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:45.457+0000 7fd8c4ddf140 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T22:56:45.785 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:45.785+0000 7fd8c4ddf140 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T22:56:46.424 INFO:tasks.ceph.cluster2.osd.0.trial082.stderr:2026-02-22T22:56:46.421+0000 7f054960f780 -1 Falling back to public interface 2026-02-22T22:56:46.429 INFO:tasks.ceph.cluster2.osd.1.trial082.stderr:2026-02-22T22:56:46.426+0000 7ffa81e2c780 -1 Falling back to public interface 2026-02-22T22:56:46.429 INFO:tasks.ceph.cluster2.osd.2.trial082.stderr:2026-02-22T22:56:46.426+0000 7f6383da3780 -1 Falling back to public interface 2026-02-22T22:56:46.791 INFO:tasks.ceph.cluster2.mgr.x.trial060.stderr:2026-02-22T22:56:46.792+0000 7fd7e19a5640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T22:56:50.914 INFO:tasks.ceph.cluster2.osd.0.trial082.stderr:2026-02-22T22:56:50.911+0000 7f054960f780 -1 osd.0 0 log_to_monitors true 2026-02-22T22:56:50.976 INFO:tasks.ceph.cluster2.osd.2.trial082.stderr:2026-02-22T22:56:50.973+0000 7f6383da3780 -1 osd.2 0 log_to_monitors true 2026-02-22T22:56:50.991 INFO:tasks.ceph.cluster2.osd.1.trial082.stderr:2026-02-22T22:56:50.989+0000 7ffa81e2c780 -1 osd.1 0 log_to_monitors true 2026-02-22T22:56:51.332 DEBUG:teuthology.orchestra.run.trial082:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:56:51.489 INFO:teuthology.misc.health.trial082.stdout: 2026-02-22T22:56:51.489 INFO:teuthology.misc.health.trial082.stdout:{"epoch":6,"fsid":"159baabb-b8a7-4828-b518-eb266dc5b284","created":"2026-02-22T22:56:43.930984+0000","modified":"2026-02-22T22:56:50.965916+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:56:44.500294+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"b46a050a-f95d-4c3b-aa52-8034685b24cb","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"7485287f-05fb-4da0-b9dc-4b04e779999d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"953c1811-ccbc-4b30-9679-3c746f040954","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:56:51.497 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:56:51.980 INFO:tasks.ceph.cluster2.osd.0.trial082.stderr:2026-02-22T22:56:51.978+0000 7f0544e4c640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T22:56:51.982 INFO:tasks.ceph.cluster2.osd.0.trial082.stderr:2026-02-22T22:56:51.980+0000 7f053cc7f640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:56:52.985 INFO:tasks.ceph.cluster2.osd.2.trial082.stderr:2026-02-22T22:56:52.983+0000 7f637fd0e640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T22:56:52.986 INFO:tasks.ceph.cluster2.osd.1.trial082.stderr:2026-02-22T22:56:52.984+0000 7ffa7d3ba640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T22:56:52.987 INFO:tasks.ceph.cluster2.osd.2.trial082.stderr:2026-02-22T22:56:52.985+0000 7f6377b41640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:56:52.988 INFO:tasks.ceph.cluster2.osd.1.trial082.stderr:2026-02-22T22:56:52.986+0000 7ffa751ed640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:56:57.800 DEBUG:teuthology.orchestra.run.trial082:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:56:57.954 INFO:teuthology.misc.health.trial082.stdout: 2026-02-22T22:56:57.954 INFO:teuthology.misc.health.trial082.stdout:{"epoch":12,"fsid":"159baabb-b8a7-4828-b518-eb266dc5b284","created":"2026-02-22T22:56:43.930984+0000","modified":"2026-02-22T22:56:56.977856+0000","last_up_change":"2026-02-22T22:56:53.975056+0000","last_in_change":"2026-02-22T22:56:44.500294+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:56:55.807109+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b46a050a-f95d-4c3b-aa52-8034685b24cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6800","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6801","nonce":4205957132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6802","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6803","nonce":4205957132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6806","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6807","nonce":4205957132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6804","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6805","nonce":4205957132}]},"public_addr":"10.20.193.82:6801/4205957132","cluster_addr":"10.20.193.82:6803/4205957132","heartbeat_back_addr":"10.20.193.82:6807/4205957132","heartbeat_front_addr":"10.20.193.82:6805/4205957132","state":["exists","up"]},{"osd":1,"uuid":"7485287f-05fb-4da0-b9dc-4b04e779999d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6808","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6810","nonce":920487848}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6813","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6815","nonce":920487848}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6821","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6823","nonce":920487848}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6817","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6819","nonce":920487848}]},"public_addr":"10.20.193.82:6810/920487848","cluster_addr":"10.20.193.82:6815/920487848","heartbeat_back_addr":"10.20.193.82:6823/920487848","heartbeat_front_addr":"10.20.193.82:6819/920487848","state":["exists","up"]},{"osd":2,"uuid":"953c1811-ccbc-4b30-9679-3c746f040954","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6809","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6811","nonce":1962775415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6812","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6814","nonce":1962775415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6820","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6822","nonce":1962775415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6816","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6818","nonce":1962775415}]},"public_addr":"10.20.193.82:6811/1962775415","cluster_addr":"10.20.193.82:6814/1962775415","heartbeat_back_addr":"10.20.193.82:6822/1962775415","heartbeat_front_addr":"10.20.193.82:6818/1962775415","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.868297+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.957575+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.971102+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:56:57.961 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T22:56:57.962 INFO:tasks.ceph:Creating RBD pool 2026-02-22T22:56:57.962 DEBUG:teuthology.orchestra.run.trial082:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-22T22:56:58.984 INFO:teuthology.orchestra.run.trial082.stderr:pool 'rbd' created 2026-02-22T22:56:58.998 DEBUG:teuthology.orchestra.run.trial082:> rbd --cluster cluster2 pool init rbd 2026-02-22T22:57:01.996 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-22T22:57:01.996 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-22T22:57:01.996 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-22T22:57:01.996 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-22T22:57:01.997 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T22:57:02.178 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:57:02.185 INFO:teuthology.orchestra.run.trial082.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6826","nonce":3964897608},{"type":"v1","addr":"10.20.193.60:6827","nonce":3964897608}]},"active_addr":"10.20.193.60:6827/3964897608","active_change":"2026-02-22T22:56:45.785238+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.60:0","nonce":550774397}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.60:0","nonce":3838691619}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.60:0","nonce":1726195131}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.60:0","nonce":3928159045}]}]} 2026-02-22T22:57:02.193 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-22T22:57:02.193 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-22T22:57:02.193 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:57:02.352 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:57:02.353 INFO:teuthology.orchestra.run.trial082.stdout:{"epoch":17,"fsid":"159baabb-b8a7-4828-b518-eb266dc5b284","created":"2026-02-22T22:56:43.930984+0000","modified":"2026-02-22T22:57:01.986784+0000","last_up_change":"2026-02-22T22:56:53.975056+0000","last_in_change":"2026-02-22T22:56:44.500294+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:56:55.807109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:56:58.122668+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b46a050a-f95d-4c3b-aa52-8034685b24cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6800","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6801","nonce":4205957132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6802","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6803","nonce":4205957132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6806","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6807","nonce":4205957132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6804","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6805","nonce":4205957132}]},"public_addr":"10.20.193.82:6801/4205957132","cluster_addr":"10.20.193.82:6803/4205957132","heartbeat_back_addr":"10.20.193.82:6807/4205957132","heartbeat_front_addr":"10.20.193.82:6805/4205957132","state":["exists","up"]},{"osd":1,"uuid":"7485287f-05fb-4da0-b9dc-4b04e779999d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6808","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6810","nonce":920487848}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6813","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6815","nonce":920487848}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6821","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6823","nonce":920487848}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6817","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6819","nonce":920487848}]},"public_addr":"10.20.193.82:6810/920487848","cluster_addr":"10.20.193.82:6815/920487848","heartbeat_back_addr":"10.20.193.82:6823/920487848","heartbeat_front_addr":"10.20.193.82:6819/920487848","state":["exists","up"]},{"osd":2,"uuid":"953c1811-ccbc-4b30-9679-3c746f040954","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6809","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6811","nonce":1962775415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6812","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6814","nonce":1962775415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6820","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6822","nonce":1962775415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6816","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6818","nonce":1962775415}]},"public_addr":"10.20.193.82:6811/1962775415","cluster_addr":"10.20.193.82:6814/1962775415","heartbeat_back_addr":"10.20.193.82:6822/1962775415","heartbeat_front_addr":"10.20.193.82:6818/1962775415","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.868297+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.957575+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.971102+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:57:02.361 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-22T22:57:02.362 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:57:02.521 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:57:02.522 INFO:teuthology.orchestra.run.trial082.stdout:{"epoch":17,"fsid":"159baabb-b8a7-4828-b518-eb266dc5b284","created":"2026-02-22T22:56:43.930984+0000","modified":"2026-02-22T22:57:01.986784+0000","last_up_change":"2026-02-22T22:56:53.975056+0000","last_in_change":"2026-02-22T22:56:44.500294+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:56:55.807109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:56:58.122668+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b46a050a-f95d-4c3b-aa52-8034685b24cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6800","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6801","nonce":4205957132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6802","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6803","nonce":4205957132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6806","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6807","nonce":4205957132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6804","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6805","nonce":4205957132}]},"public_addr":"10.20.193.82:6801/4205957132","cluster_addr":"10.20.193.82:6803/4205957132","heartbeat_back_addr":"10.20.193.82:6807/4205957132","heartbeat_front_addr":"10.20.193.82:6805/4205957132","state":["exists","up"]},{"osd":1,"uuid":"7485287f-05fb-4da0-b9dc-4b04e779999d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6808","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6810","nonce":920487848}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6813","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6815","nonce":920487848}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6821","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6823","nonce":920487848}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6817","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6819","nonce":920487848}]},"public_addr":"10.20.193.82:6810/920487848","cluster_addr":"10.20.193.82:6815/920487848","heartbeat_back_addr":"10.20.193.82:6823/920487848","heartbeat_front_addr":"10.20.193.82:6819/920487848","state":["exists","up"]},{"osd":2,"uuid":"953c1811-ccbc-4b30-9679-3c746f040954","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6809","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6811","nonce":1962775415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6812","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6814","nonce":1962775415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6820","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6822","nonce":1962775415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6816","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6818","nonce":1962775415}]},"public_addr":"10.20.193.82:6811/1962775415","cluster_addr":"10.20.193.82:6814/1962775415","heartbeat_back_addr":"10.20.193.82:6822/1962775415","heartbeat_front_addr":"10.20.193.82:6818/1962775415","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.868297+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.957575+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.971102+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:57:02.530 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-22T22:57:02.531 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-22T22:57:02.531 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-22T22:57:02.620 INFO:teuthology.orchestra.run.trial082.stdout:38654705668 2026-02-22T22:57:02.620 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T22:57:02.621 INFO:teuthology.orchestra.run.trial082.stdout:34359738372 2026-02-22T22:57:02.621 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T22:57:02.622 INFO:teuthology.orchestra.run.trial082.stdout:38654705669 2026-02-22T22:57:02.622 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T22:57:02.822 INFO:teuthology.orchestra.run.trial082.stdout:38654705667 2026-02-22T22:57:02.823 INFO:teuthology.orchestra.run.trial082.stdout:34359738371 2026-02-22T22:57:02.831 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705667 for osd.2 2026-02-22T22:57:02.832 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-22T22:57:02.864 INFO:teuthology.orchestra.run.trial082.stdout:38654705668 2026-02-22T22:57:02.873 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705668 for osd.1 2026-02-22T22:57:03.832 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T22:57:03.833 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T22:57:03.874 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T22:57:03.993 INFO:teuthology.orchestra.run.trial082.stdout:38654705668 2026-02-22T22:57:03.997 INFO:teuthology.orchestra.run.trial082.stdout:34359738373 2026-02-22T22:57:04.003 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705668 for osd.2 2026-02-22T22:57:04.003 DEBUG:teuthology.parallel:result is None 2026-02-22T22:57:04.006 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738373 for osd.0 2026-02-22T22:57:04.006 DEBUG:teuthology.parallel:result is None 2026-02-22T22:57:04.032 INFO:teuthology.orchestra.run.trial082.stdout:38654705669 2026-02-22T22:57:04.042 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705669 for osd.1 2026-02-22T22:57:04.042 DEBUG:teuthology.parallel:result is None 2026-02-22T22:57:04.042 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T22:57:04.042 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:57:04.239 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:57:04.240 INFO:teuthology.orchestra.run.trial082.stderr:dumped all 2026-02-22T22:57:04.247 INFO:teuthology.orchestra.run.trial082.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2026-02-22T22:57:03.792581+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82684,"kb_used_data":1584,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283032836,"statfs":{"total":289910292480,"available":289825624064,"internally_reserved":0,"allocated":1622016,"data_stored":1302985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.806250"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.989573+0000","last_change":"2026-02-22T22:57:01.989798+0000","last_active":"2026-02-22T22:57:01.989573+0000","last_peered":"2026-02-22T22:57:01.989573+0000","last_clean":"2026-02-22T22:57:01.989573+0000","last_became_active":"2026-02-22T22:56:59.987009+0000","last_became_peered":"2026-02-22T22:56:59.987009+0000","last_unstale":"2026-02-22T22:57:01.989573+0000","last_undegraded":"2026-02-22T22:57:01.989573+0000","last_fullsized":"2026-02-22T22:57:01.989573+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:37.582516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064088199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.989340+0000","last_change":"2026-02-22T22:57:01.989565+0000","last_active":"2026-02-22T22:57:01.989340+0000","last_peered":"2026-02-22T22:57:01.989340+0000","last_clean":"2026-02-22T22:57:01.989340+0000","last_became_active":"2026-02-22T22:56:59.987115+0000","last_became_peered":"2026-02-22T22:56:59.987115+0000","last_unstale":"2026-02-22T22:57:01.989340+0000","last_undegraded":"2026-02-22T22:57:01.989340+0000","last_fullsized":"2026-02-22T22:57:01.989340+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:03:17.983122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000448434,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.989829+0000","last_change":"2026-02-22T22:57:01.989987+0000","last_active":"2026-02-22T22:57:01.989829+0000","last_peered":"2026-02-22T22:57:01.989829+0000","last_clean":"2026-02-22T22:57:01.989829+0000","last_became_active":"2026-02-22T22:56:59.987092+0000","last_became_peered":"2026-02-22T22:56:59.987092+0000","last_unstale":"2026-02-22T22:57:01.989829+0000","last_undegraded":"2026-02-22T22:57:01.989829+0000","last_fullsized":"2026-02-22T22:57:01.989829+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:29:49.602573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000831597,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.989311+0000","last_change":"2026-02-22T22:57:01.989535+0000","last_active":"2026-02-22T22:57:01.989311+0000","last_peered":"2026-02-22T22:57:01.989311+0000","last_clean":"2026-02-22T22:57:01.989311+0000","last_became_active":"2026-02-22T22:56:59.987136+0000","last_became_peered":"2026-02-22T22:56:59.987136+0000","last_unstale":"2026-02-22T22:57:01.989311+0000","last_undegraded":"2026-02-22T22:57:01.989311+0000","last_fullsized":"2026-02-22T22:57:01.989311+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:55.509626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000443876,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.991812+0000","last_change":"2026-02-22T22:57:01.991533+0000","last_active":"2026-02-22T22:57:01.991812+0000","last_peered":"2026-02-22T22:57:01.991812+0000","last_clean":"2026-02-22T22:57:01.991812+0000","last_became_active":"2026-02-22T22:56:59.987223+0000","last_became_peered":"2026-02-22T22:56:59.987223+0000","last_unstale":"2026-02-22T22:57:01.991812+0000","last_undegraded":"2026-02-22T22:57:01.991812+0000","last_fullsized":"2026-02-22T22:57:01.991812+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:21:47.758782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051754199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:02.043521+0000","last_change":"2026-02-22T22:57:02.043669+0000","last_active":"2026-02-22T22:57:02.043521+0000","last_peered":"2026-02-22T22:57:02.043521+0000","last_clean":"2026-02-22T22:57:02.043521+0000","last_became_active":"2026-02-22T22:56:59.987301+0000","last_became_peered":"2026-02-22T22:56:59.987301+0000","last_unstale":"2026-02-22T22:57:02.043521+0000","last_undegraded":"2026-02-22T22:57:02.043521+0000","last_fullsized":"2026-02-22T22:57:02.043521+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:02:11.006528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029387700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:02.043491+0000","last_change":"2026-02-22T22:57:02.043628+0000","last_active":"2026-02-22T22:57:02.043491+0000","last_peered":"2026-02-22T22:57:02.043491+0000","last_clean":"2026-02-22T22:57:02.043491+0000","last_became_active":"2026-02-22T22:56:59.987306+0000","last_became_peered":"2026-02-22T22:56:59.987306+0000","last_unstale":"2026-02-22T22:57:02.043491+0000","last_undegraded":"2026-02-22T22:57:02.043491+0000","last_fullsized":"2026-02-22T22:57:02.043491+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:00:55.873154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026463300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.989681+0000","last_change":"2026-02-22T22:57:01.989963+0000","last_active":"2026-02-22T22:57:01.989681+0000","last_peered":"2026-02-22T22:57:01.989681+0000","last_clean":"2026-02-22T22:57:01.989681+0000","last_became_active":"2026-02-22T22:56:59.986733+0000","last_became_peered":"2026-02-22T22:56:59.986733+0000","last_unstale":"2026-02-22T22:57:01.989681+0000","last_undegraded":"2026-02-22T22:57:01.989681+0000","last_fullsized":"2026-02-22T22:57:01.989681+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:33:04.386924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080243299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'72","reported_seq":110,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.988896+0000","last_change":"2026-02-22T22:56:56.981052+0000","last_active":"2026-02-22T22:57:01.988896+0000","last_peered":"2026-02-22T22:57:01.988896+0000","last_clean":"2026-02-22T22:57:01.988896+0000","last_became_active":"2026-02-22T22:56:56.980851+0000","last_became_peered":"2026-02-22T22:56:56.980851+0000","last_unstale":"2026-02-22T22:57:01.988896+0000","last_undegraded":"2026-02-22T22:57:01.988896+0000","last_fullsized":"2026-02-22T22:57:01.988896+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_clean_scrub_stamp":"2026-02-22T22:56:55.976608+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:21:14.696753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27104,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344736,"statfs":{"total":96636764160,"available":96609009664,"internally_reserved":0,"allocated":139264,"data_stored":39554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27884,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94343956,"statfs":{"total":96636764160,"available":96608210944,"internally_reserved":0,"allocated":737280,"data_stored":629930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27696,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344144,"statfs":{"total":96636764160,"available":96608403456,"internally_reserved":0,"allocated":745472,"data_stored":633501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:57:04.248 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:57:04.402 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:57:04.403 INFO:teuthology.orchestra.run.trial082.stderr:dumped all 2026-02-22T22:57:04.410 INFO:teuthology.orchestra.run.trial082.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2026-02-22T22:57:03.792581+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82684,"kb_used_data":1584,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283032836,"statfs":{"total":289910292480,"available":289825624064,"internally_reserved":0,"allocated":1622016,"data_stored":1302985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.806250"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.989573+0000","last_change":"2026-02-22T22:57:01.989798+0000","last_active":"2026-02-22T22:57:01.989573+0000","last_peered":"2026-02-22T22:57:01.989573+0000","last_clean":"2026-02-22T22:57:01.989573+0000","last_became_active":"2026-02-22T22:56:59.987009+0000","last_became_peered":"2026-02-22T22:56:59.987009+0000","last_unstale":"2026-02-22T22:57:01.989573+0000","last_undegraded":"2026-02-22T22:57:01.989573+0000","last_fullsized":"2026-02-22T22:57:01.989573+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:37.582516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064088199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.989340+0000","last_change":"2026-02-22T22:57:01.989565+0000","last_active":"2026-02-22T22:57:01.989340+0000","last_peered":"2026-02-22T22:57:01.989340+0000","last_clean":"2026-02-22T22:57:01.989340+0000","last_became_active":"2026-02-22T22:56:59.987115+0000","last_became_peered":"2026-02-22T22:56:59.987115+0000","last_unstale":"2026-02-22T22:57:01.989340+0000","last_undegraded":"2026-02-22T22:57:01.989340+0000","last_fullsized":"2026-02-22T22:57:01.989340+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:03:17.983122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000448434,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.989829+0000","last_change":"2026-02-22T22:57:01.989987+0000","last_active":"2026-02-22T22:57:01.989829+0000","last_peered":"2026-02-22T22:57:01.989829+0000","last_clean":"2026-02-22T22:57:01.989829+0000","last_became_active":"2026-02-22T22:56:59.987092+0000","last_became_peered":"2026-02-22T22:56:59.987092+0000","last_unstale":"2026-02-22T22:57:01.989829+0000","last_undegraded":"2026-02-22T22:57:01.989829+0000","last_fullsized":"2026-02-22T22:57:01.989829+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:29:49.602573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000831597,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.989311+0000","last_change":"2026-02-22T22:57:01.989535+0000","last_active":"2026-02-22T22:57:01.989311+0000","last_peered":"2026-02-22T22:57:01.989311+0000","last_clean":"2026-02-22T22:57:01.989311+0000","last_became_active":"2026-02-22T22:56:59.987136+0000","last_became_peered":"2026-02-22T22:56:59.987136+0000","last_unstale":"2026-02-22T22:57:01.989311+0000","last_undegraded":"2026-02-22T22:57:01.989311+0000","last_fullsized":"2026-02-22T22:57:01.989311+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:55.509626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000443876,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.991812+0000","last_change":"2026-02-22T22:57:01.991533+0000","last_active":"2026-02-22T22:57:01.991812+0000","last_peered":"2026-02-22T22:57:01.991812+0000","last_clean":"2026-02-22T22:57:01.991812+0000","last_became_active":"2026-02-22T22:56:59.987223+0000","last_became_peered":"2026-02-22T22:56:59.987223+0000","last_unstale":"2026-02-22T22:57:01.991812+0000","last_undegraded":"2026-02-22T22:57:01.991812+0000","last_fullsized":"2026-02-22T22:57:01.991812+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:21:47.758782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051754199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:02.043521+0000","last_change":"2026-02-22T22:57:02.043669+0000","last_active":"2026-02-22T22:57:02.043521+0000","last_peered":"2026-02-22T22:57:02.043521+0000","last_clean":"2026-02-22T22:57:02.043521+0000","last_became_active":"2026-02-22T22:56:59.987301+0000","last_became_peered":"2026-02-22T22:56:59.987301+0000","last_unstale":"2026-02-22T22:57:02.043521+0000","last_undegraded":"2026-02-22T22:57:02.043521+0000","last_fullsized":"2026-02-22T22:57:02.043521+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:02:11.006528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029387700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:02.043491+0000","last_change":"2026-02-22T22:57:02.043628+0000","last_active":"2026-02-22T22:57:02.043491+0000","last_peered":"2026-02-22T22:57:02.043491+0000","last_clean":"2026-02-22T22:57:02.043491+0000","last_became_active":"2026-02-22T22:56:59.987306+0000","last_became_peered":"2026-02-22T22:56:59.987306+0000","last_unstale":"2026-02-22T22:57:02.043491+0000","last_undegraded":"2026-02-22T22:57:02.043491+0000","last_fullsized":"2026-02-22T22:57:02.043491+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:00:55.873154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026463300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.989681+0000","last_change":"2026-02-22T22:57:01.989963+0000","last_active":"2026-02-22T22:57:01.989681+0000","last_peered":"2026-02-22T22:57:01.989681+0000","last_clean":"2026-02-22T22:57:01.989681+0000","last_became_active":"2026-02-22T22:56:59.986733+0000","last_became_peered":"2026-02-22T22:56:59.986733+0000","last_unstale":"2026-02-22T22:57:01.989681+0000","last_undegraded":"2026-02-22T22:57:01.989681+0000","last_fullsized":"2026-02-22T22:57:01.989681+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:33:04.386924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080243299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'72","reported_seq":110,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:57:01.988896+0000","last_change":"2026-02-22T22:56:56.981052+0000","last_active":"2026-02-22T22:57:01.988896+0000","last_peered":"2026-02-22T22:57:01.988896+0000","last_clean":"2026-02-22T22:57:01.988896+0000","last_became_active":"2026-02-22T22:56:56.980851+0000","last_became_peered":"2026-02-22T22:56:56.980851+0000","last_unstale":"2026-02-22T22:57:01.988896+0000","last_undegraded":"2026-02-22T22:57:01.988896+0000","last_fullsized":"2026-02-22T22:57:01.988896+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_clean_scrub_stamp":"2026-02-22T22:56:55.976608+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:21:14.696753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27104,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344736,"statfs":{"total":96636764160,"available":96609009664,"internally_reserved":0,"allocated":139264,"data_stored":39554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27884,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94343956,"statfs":{"total":96636764160,"available":96608210944,"internally_reserved":0,"allocated":737280,"data_stored":629930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27696,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344144,"statfs":{"total":96636764160,"available":96608403456,"internally_reserved":0,"allocated":745472,"data_stored":633501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:57:04.411 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T22:57:04.411 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-22T22:57:04.411 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-22T22:57:04.411 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-22T22:57:04.584 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T22:57:04.587 INFO:teuthology.orchestra.run.trial082.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T22:57:04.592 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-22T22:57:04.593 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T22:57:04.610 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T22:57:04.611 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial082.front.sepia.ceph.com 2026-02-22T22:57:04.611 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:04.809 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.4] 2026-02-22T22:57:04.809 INFO:teuthology.orchestra.run.trial082.stdout: key = AQBxiZtpEsHJOhAAL8tq3E1130CW24LT6sIsFw== 2026-02-22T22:57:04.809 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:04.809 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:04.810 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.4 2026-02-22T22:57:04.818 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:05.013 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.5] 2026-02-22T22:57:05.013 INFO:teuthology.orchestra.run.trial082.stdout: key = AQByiZtp4ig8BBAAG5d4FQ87VlosT+pcoJAksw== 2026-02-22T22:57:05.013 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:05.013 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:05.014 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.5 2026-02-22T22:57:05.023 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:05.217 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.6] 2026-02-22T22:57:05.217 INFO:teuthology.orchestra.run.trial082.stdout: key = AQByiZtp9ZtFCRAAxggF6Tys+P6+KizI+pZpXw== 2026-02-22T22:57:05.218 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:05.218 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:05.218 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.6 2026-02-22T22:57:05.227 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:05.422 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.4] 2026-02-22T22:57:05.422 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCbiZtp84nmAxAAgArySjgCAQ034NH9rYvm/Q== 2026-02-22T22:57:05.422 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:05.422 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:05.423 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.4 2026-02-22T22:57:05.431 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:05.629 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.5] 2026-02-22T22:57:05.629 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCbiZtpHVEVCRAASwKOV1V4bQm71tAPJt0Dbg== 2026-02-22T22:57:05.629 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:05.629 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:05.629 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.5 2026-02-22T22:57:05.639 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:05.836 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.6] 2026-02-22T22:57:05.836 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCbiZtpSKAwDhAAoIePBbyFIkIdOqDxr3oCvA== 2026-02-22T22:57:05.836 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:05.837 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:05.837 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.6 2026-02-22T22:57:05.846 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T22:57:05.861 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T22:57:05.862 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial082.front.sepia.ceph.com 2026-02-22T22:57:05.862 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:06.060 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror] 2026-02-22T22:57:06.060 INFO:teuthology.orchestra.run.trial082.stdout: key = AQBxiZtpWDhEIRAA9LzFBw6JUgRNP8NIbjBvdg== 2026-02-22T22:57:06.061 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T22:57:06.061 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:06.061 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror 2026-02-22T22:57:06.070 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:06.265 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.0] 2026-02-22T22:57:06.265 INFO:teuthology.orchestra.run.trial082.stdout: key = AQBxiZtpTpJiJhAAn1EVPKVZYet8RQ2sL3y3Zw== 2026-02-22T22:57:06.265 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:06.265 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:06.265 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.0 2026-02-22T22:57:06.274 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:06.470 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.1] 2026-02-22T22:57:06.470 INFO:teuthology.orchestra.run.trial082.stdout: key = AQBxiZtpHjiDKxAACe6nwhQ9rgCwtiZ22bXINQ== 2026-02-22T22:57:06.470 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:06.471 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:06.471 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.1 2026-02-22T22:57:06.479 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:06.676 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.2] 2026-02-22T22:57:06.677 INFO:teuthology.orchestra.run.trial082.stdout: key = AQBxiZtpOIePMBAAN40+eK2x5napvEudATk2PQ== 2026-02-22T22:57:06.677 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:06.677 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:06.677 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.2 2026-02-22T22:57:06.686 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:06.882 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.3] 2026-02-22T22:57:06.882 INFO:teuthology.orchestra.run.trial082.stdout: key = AQBxiZtp7u6xNRAAe8S0enaPsfdlKELGKMMQkg== 2026-02-22T22:57:06.882 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:06.882 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:06.882 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.3 2026-02-22T22:57:06.891 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:07.095 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror] 2026-02-22T22:57:07.096 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtp7HFoJRAAxSKsJ9KLDt+ukcfRSAwwBA== 2026-02-22T22:57:07.096 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T22:57:07.096 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:07.096 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror 2026-02-22T22:57:07.105 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:07.305 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.0] 2026-02-22T22:57:07.305 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtpRq5oKhAAG0UU87I7ziy8MfMM9WTNaA== 2026-02-22T22:57:07.305 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:07.305 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:07.305 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.0 2026-02-22T22:57:07.314 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:07.511 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.1] 2026-02-22T22:57:07.511 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtpzGaoLxAAewwQAXEYciQyIiUzVoe4mw== 2026-02-22T22:57:07.511 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:07.511 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:07.511 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.1 2026-02-22T22:57:07.521 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:07.716 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.2] 2026-02-22T22:57:07.717 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtp4ZnMNBAAldjijeFiEcDO3BZvh8ZWOg== 2026-02-22T22:57:07.717 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:07.717 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:07.717 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.2 2026-02-22T22:57:07.727 DEBUG:teuthology.orchestra.run.trial082:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:57:07.923 INFO:teuthology.orchestra.run.trial082.stdout:[client.mirror.3] 2026-02-22T22:57:07.923 INFO:teuthology.orchestra.run.trial082.stdout: key = AQCaiZtpSNrSORAA947XpKn4pjJ7ccD9j0MVzg== 2026-02-22T22:57:07.923 INFO:teuthology.orchestra.run.trial082.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:57:07.923 INFO:teuthology.orchestra.run.trial082.stdout: caps osd = "profile rbd" 2026-02-22T22:57:07.924 INFO:teuthology.orchestra.run.trial082.stderr:updated caps for client.mirror.3 2026-02-22T22:57:07.933 INFO:teuthology.run_tasks:Running task workunit... 2026-02-22T22:57:07.951 INFO:tasks.workunit:Pulling workunits from ref f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:57:07.952 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-22T22:57:07.952 DEBUG:teuthology.orchestra.run.trial082:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:57:07.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:57:07.968 INFO:teuthology.orchestra.run.trial082.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-22T22:57:07.968 DEBUG:teuthology.orchestra.run.trial082:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:57:08.025 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:57:08.025 DEBUG:teuthology.orchestra.run.trial082:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-22T22:57:08.080 INFO:tasks.workunit:timeout=3h 2026-02-22T22:57:08.080 INFO:tasks.workunit:cleanup=True 2026-02-22T22:57:08.081 DEBUG:teuthology.orchestra.run.trial082:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:57:08.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-22T22:57:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Note: switching to 'f90f4eaebb28b619cf76e80cbbe997656da69579'. 2026-02-22T22:57:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T22:57:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:state without impacting any branches by switching back to a branch. 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: git switch -c 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Or undo this operation with: 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T22:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: git switch - 2026-02-22T22:57:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T22:57:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-22T22:57:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T22:57:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:HEAD is now at f90f4eaebb2 Merge branch 'wip-74712-squid' of https://github.com/idryomov/ceph into wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:57:28.284 DEBUG:teuthology.orchestra.run.trial082:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-22T22:57:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-22T22:57:28.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T22:57:28.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-22T22:57:28.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-22T22:57:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-22T22:57:28.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T22:57:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T22:57:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-22T22:57:28.523 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T22:57:28.527 DEBUG:teuthology.orchestra.run.trial082:> set -ex 2026-02-22T22:57:28.528 DEBUG:teuthology.orchestra.run.trial082:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-22T22:57:28.585 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror.sh on cluster1.client.mirror... 2026-02-22T22:57:28.586 INFO:tasks.workunit:Running workunit rbd/rbd_mirror.sh... 2026-02-22T22:57:28.586 DEBUG:teuthology.orchestra.run.trial082:workunit test rbd/rbd_mirror.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f90f4eaebb28b619cf76e80cbbe997656da69579 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' RBD_IMAGE_FEATURES=layering,exclusive-lock RBD_MIRROR_CONFIG_KEY=1 RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-22T22:57:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -gt 0 ']' 2026-02-22T22:57:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot '!=' snapshot ']' 2026-02-22T22:57:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-22T22:57:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-22T22:57:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ type xmlstarlet 2026-02-22T22:57:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ XMLSTARLET=xmlstarlet 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ RBD_MIRROR_INSTANCES=4 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CLUSTER1=cluster1 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CLUSTER2=cluster2 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ POOL=mirror 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ PARENT_POOL=mirror_parent 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ NS1=ns1 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ NS2=ns2 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ TEMPDIR= 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ID=mirror 2026-02-22T22:57:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ RBD_MIRROR_MODE=snapshot 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ MIRROR_POOL_MODE=pool 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' snapshot = snapshot ']' 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ MIRROR_POOL_MODE=image 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS='--id mirror' 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ LAST_MIRROR_INSTANCE=3 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-22T22:57:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_BIN=. 2026-02-22T22:57:28.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_SRC=. 2026-02-22T22:57:28.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-22T22:57:28.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 -gt 0 ']' 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ setup 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local c 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ setup_tempdir 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mktemp -d 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ TEMPDIR=/tmp/tmp.VQMrvUqClS 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z 1 ']' 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ setup_pools cluster1 cluster2 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:28.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local remote_cluster=cluster2 2026-02-22T22:57:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mon_map_file 2026-02-22T22:57:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mon_addr 2026-02-22T22:57:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local admin_key_file 2026-02-22T22:57:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local uuid 2026-02-22T22:57:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-22T22:57:28.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:pool 'mirror' already exists 2026-02-22T22:57:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:28.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-22T22:57:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:pool 'mirror_parent' already exists 2026-02-22T22:57:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-22T22:57:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-22T22:57:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 1 ']' 2026-02-22T22:57:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ PEER_CLUSTER_SUFFIX=-DNE 2026-02-22T22:57:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1-DNE mirror image 2026-02-22T22:57:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-22T22:57:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-22T22:57:36.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-22T22:57:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-22T22:57:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2026-02-22T22:57:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-22T22:57:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 image 2026-02-22T22:57:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z ']' 2026-02-22T22:57:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z 1 ']' 2026-02-22T22:57:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mon_map_file=/tmp/tmp.VQMrvUqClS/cluster2.monmap 2026-02-22T22:57:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster2 mon getmap 2026-02-22T22:57:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:got monmap epoch 1 2026-02-22T22:57:36.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ monmaptool --print /tmp/tmp.VQMrvUqClS/cluster2.monmap 2026-02-22T22:57:36.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -E 'mon\.' 2026-02-22T22:57:36.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ head -n 1 2026-02-22T22:57:36.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' 2026-02-22T22:57:36.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -E 's/\/[0-9]+//g' 2026-02-22T22:57:36.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mon_addr='[v2:10.20.193.82:3300,v1:10.20.193.82:6789]' 2026-02-22T22:57:36.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_key_file=/tmp/tmp.VQMrvUqClS/cluster2.client.mirror.key 2026-02-22T22:57:36.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:36.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster2 auth get-key client.mirror 2026-02-22T22:57:36.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ peer_add cluster1 mirror client.mirror@cluster2-DNE '' --remote-mon-host '[v2:10.20.193.82:3300,v1:10.20.193.82:6789]' --remote-key-file /tmp/tmp.VQMrvUqClS/cluster2.client.mirror.key 2026-02-22T22:57:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local client_cluster=client.mirror@cluster2-DNE 2026-02-22T22:57:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local remote_cluster=cluster2-DNE 2026-02-22T22:57:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local uuid_var_name 2026-02-22T22:57:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local error_code 2026-02-22T22:57:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local peer_uuid 2026-02-22T22:57:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T22:57:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:57:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror client.mirror@cluster2-DNE --remote-mon-host '[v2:10.20.193.82:3300,v1:10.20.193.82:6789]' --remote-key-file /tmp/tmp.VQMrvUqClS/cluster2.client.mirror.key 2026-02-22T22:57:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ peer_uuid=70a926a6-5554-4bd7-a6ec-e3f48f4efa7b 2026-02-22T22:57:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T22:57:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:57:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T22:57:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 -eq 0 2026-02-22T22:57:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ peer_add cluster1 mirror_parent client.mirror@cluster2-DNE uuid 2026-02-22T22:57:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local client_cluster=client.mirror@cluster2-DNE 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local remote_cluster=cluster2-DNE 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local uuid_var_name 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n uuid ']' 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ uuid_var_name=uuid 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local error_code 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local peer_uuid 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:57:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent client.mirror@cluster2-DNE 2026-02-22T22:57:36.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ peer_uuid=a5da3887-dec5-4279-9da0-8412a090cb5c 2026-02-22T22:57:36.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T22:57:36.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:57:36.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T22:57:36.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 -eq 0 2026-02-22T22:57:36.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n uuid ']' 2026-02-22T22:57:36.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval uuid=a5da3887-dec5-4279-9da0-8412a090cb5c 2026-02-22T22:57:36.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ uuid=a5da3887-dec5-4279-9da0-8412a090cb5c 2026-02-22T22:57:36.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:36.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:36.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror pool peer set mirror_parent a5da3887-dec5-4279-9da0-8412a090cb5c mon-host '[v2:10.20.193.82:3300,v1:10.20.193.82:6789]' 2026-02-22T22:57:36.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:36.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror pool peer set mirror_parent a5da3887-dec5-4279-9da0-8412a090cb5c key-file /tmp/tmp.VQMrvUqClS/cluster2.client.mirror.key 2026-02-22T22:57:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ setup_pools cluster2 cluster1 2026-02-22T22:57:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local remote_cluster=cluster1 2026-02-22T22:57:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mon_map_file 2026-02-22T22:57:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mon_addr 2026-02-22T22:57:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local admin_key_file 2026-02-22T22:57:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local uuid 2026-02-22T22:57:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-22T22:57:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:pool 'mirror' already exists 2026-02-22T22:57:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-22T22:57:38.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:pool 'mirror_parent' already exists 2026-02-22T22:57:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-22T22:57:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-22T22:57:43.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 1 ']' 2026-02-22T22:57:43.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ PEER_CLUSTER_SUFFIX=-DNE 2026-02-22T22:57:43.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:43.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2-DNE mirror image 2026-02-22T22:57:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-22T22:57:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-22T22:57:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-22T22:57:44.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-22T22:57:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2026-02-22T22:57:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-22T22:57:44.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 image 2026-02-22T22:57:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z ']' 2026-02-22T22:57:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z 1 ']' 2026-02-22T22:57:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mon_map_file=/tmp/tmp.VQMrvUqClS/cluster1.monmap 2026-02-22T22:57:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster1 mon getmap 2026-02-22T22:57:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:got monmap epoch 1 2026-02-22T22:57:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ monmaptool --print /tmp/tmp.VQMrvUqClS/cluster1.monmap 2026-02-22T22:57:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -E 'mon\.' 2026-02-22T22:57:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ head -n 1 2026-02-22T22:57:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' 2026-02-22T22:57:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -E 's/\/[0-9]+//g' 2026-02-22T22:57:44.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mon_addr='[v2:10.20.193.60:3300,v1:10.20.193.60:6789]' 2026-02-22T22:57:44.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_key_file=/tmp/tmp.VQMrvUqClS/cluster1.client.mirror.key 2026-02-22T22:57:44.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster1 auth get-key client.mirror 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ peer_add cluster2 mirror client.mirror@cluster1-DNE '' --remote-mon-host '[v2:10.20.193.60:3300,v1:10.20.193.60:6789]' --remote-key-file /tmp/tmp.VQMrvUqClS/cluster1.client.mirror.key 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local client_cluster=client.mirror@cluster1-DNE 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local remote_cluster=cluster1-DNE 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local uuid_var_name 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local error_code 2026-02-22T22:57:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local peer_uuid 2026-02-22T22:57:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T22:57:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:57:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror client.mirror@cluster1-DNE --remote-mon-host '[v2:10.20.193.60:3300,v1:10.20.193.60:6789]' --remote-key-file /tmp/tmp.VQMrvUqClS/cluster1.client.mirror.key 2026-02-22T22:57:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ peer_uuid=6a3341eb-d62f-469b-8fe4-d32d7815b1ae 2026-02-22T22:57:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T22:57:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:57:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T22:57:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 -eq 0 2026-02-22T22:57:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ peer_add cluster2 mirror_parent client.mirror@cluster1-DNE uuid 2026-02-22T22:57:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local client_cluster=client.mirror@cluster1-DNE 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local remote_cluster=cluster1-DNE 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local uuid_var_name 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n uuid ']' 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ uuid_var_name=uuid 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local error_code 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local peer_uuid 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T22:57:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:57:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent client.mirror@cluster1-DNE 2026-02-22T22:57:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ peer_uuid=aaaa9fd4-e695-45e1-ac78-444e552b7f59 2026-02-22T22:57:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T22:57:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:57:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T22:57:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 -eq 0 2026-02-22T22:57:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n uuid ']' 2026-02-22T22:57:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval uuid=aaaa9fd4-e695-45e1-ac78-444e552b7f59 2026-02-22T22:57:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ uuid=aaaa9fd4-e695-45e1-ac78-444e552b7f59 2026-02-22T22:57:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror pool peer set mirror_parent aaaa9fd4-e695-45e1-ac78-444e552b7f59 mon-host '[v2:10.20.193.60:3300,v1:10.20.193.60:6789]' 2026-02-22T22:57:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T22:57:44.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror pool peer set mirror_parent aaaa9fd4-e695-45e1-ac78-444e552b7f59 key-file /tmp/tmp.VQMrvUqClS/cluster1.client.mirror.key 2026-02-22T22:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: add image and test replay' 2026-02-22T22:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T22:57:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T22:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 22:57:44 TEST: add image and test replay 2026-02-22T22:57:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 22:57:44 TEST: add image and test replay 2026-02-22T22:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirrors cluster1 2026-02-22T22:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:57:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:0 2026-02-22T22:57:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:57:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:57:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:57:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:57:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:57:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:57:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:57:44.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:57:44.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:44.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:57:44.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:44.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:57:44.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:57:44.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:44.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:57:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:1 2026-02-22T22:57:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:57:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:57:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:57:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:2 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:57:44.822 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:57:44.822 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:44.822 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:57:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:3 2026-02-22T22:57:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:57:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:57:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:57:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image=test 2026-02-22T22:57:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 2026-02-22T22:57:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T22:57:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test 2026-02-22T22:57:44.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:57:44.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:44.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T22:57:44.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:44.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-22T22:57:44.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T22:57:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-22T22:57:45.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T22:57:45.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-22T22:57:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test': 2026-02-22T22:57:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T22:57:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T22:57:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T22:57:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 10b8ad6ebe7d 2026-02-22T22:57:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.10b8ad6ebe7d 2026-02-22T22:57:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T22:57:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T22:57:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T22:57:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T22:57:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 22:57:44 2026 2026-02-22T22:57:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 22:57:44 2026 2026-02-22T22:57:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 22:57:44 2026 2026-02-22T22:57:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T22:57:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T22:57:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:57:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T22:57:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_image_meta cluster2 mirror test key1 value1 2026-02-22T22:57:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local key=key1 2026-02-22T22:57:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=value1 2026-02-22T22:57:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 image-meta set mirror/test key1 value1 2026-02-22T22:57:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_image_meta cluster2 mirror test key2 value2 2026-02-22T22:57:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local key=key2 2026-02-22T22:57:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=value2 2026-02-22T22:57:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 image-meta set mirror/test key2 value2 2026-02-22T22:57:45.875 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:57:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:57:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:57:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:57:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:57:45.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:57:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:57:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:57:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:57:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:57:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:57:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:57:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:57:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:57:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:57:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:57:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:57:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:57:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:57:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:57:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:57:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:57:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:57:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:57:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:57:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:57:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:57:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:57:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:57:46.885 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:57:46.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:46.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:57:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:57:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:57:46.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:57:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:57:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:57:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:57:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:57:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:57:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:57:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:57:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:57:46.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:57:46.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:57:46.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:57:46.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:57:46.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:57:46.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:57:46.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:57:46.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:46.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T22:57:46.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T22:57:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:57:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:57:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:57:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:57:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:57:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:57:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:57:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:57:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:57:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:57:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T22:57:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:57:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:57:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:57:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:57:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:57:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T22:57:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T22:57:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:57:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:57:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:57:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:57:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T22:57:47.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:57:47.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:57:47.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:57:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T22:57:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T22:57:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:57:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:57:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:57:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:57:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:57:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:57:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:57:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:57:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:57:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:57:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T22:57:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:57:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:57:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:57:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:57:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:57:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T22:57:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T22:57:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:57:47.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:57:47.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:57:47.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:57:47.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T22:57:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:57:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:57:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:57:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:57:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T22:57:48.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:57:48.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:57:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:57:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:57:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:57:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:57:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:57:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:48.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:48.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:48.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:57:48.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:57:48.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:57:48.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:57:48.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:57:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:57:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:57:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:57:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:57:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:57:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:57:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test 100 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T22:57:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:57:48.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:57:48.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:57:48.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 11111.2 bytes/sec: 43 MiB/s 2026-02-22T22:57:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T22:57:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:57:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:57:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:57:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T22:57:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:48.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:48.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T22:57:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 4 2026-02-22T22:57:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T22:57:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T22:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T22:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:57:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:57:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T22:57:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T22:57:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:57:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:57:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:57:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:57:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T22:57:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:57:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:57:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=4 2026-02-22T22:57:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T22:57:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:57:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T22:57:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:57:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:57:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T22:57:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:57:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:57:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=4 2026-02-22T22:57:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 4 = 4 2026-02-22T22:57:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T22:57:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:57:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:57:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:57:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:57:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:57:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:57:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:57:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:57:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:57:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:57:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:57:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:57:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:57:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:57:50.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:57:50.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:57:50.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:57:50.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801065,"replay_state":"idle"} 2026-02-22T22:57:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:57:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:57:47' 2026-02-22T22:57:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:57:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:57:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:57:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801065,"replay_state":"idle"} 2026-02-22T22:57:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:57:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:57:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:57:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:57:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:57:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:57:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:57:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:57:47' 2026-02-22T22:57:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:57:47' 2026-02-22T22:57:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:57:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:57:47' 2026-02-22T22:57:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:57:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:57:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:57:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING142734277mirror.0trial08219.2.3-689-gf90f4eaetrueOK42794283mirror.1trial08219.2.3-689-gf90f4eaefalseOK42854289mirror.2trial08219.2.3-689-gf90f4eaefalseOK42944300mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801065,"replay_state":"idle"}42734277mirror.0trial0822026-02-22 22:57:47' 2026-02-22T22:57:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:57:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:57:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:57:47' 2026-02-22T22:57:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:57:50.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:57:50.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:57:50.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:57:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:57:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z '' ']' 2026-02-22T22:57:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test down+unknown 2026-02-22T22:57:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=down+unknown 2026-02-22T22:57:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:57:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:57:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:57:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:57:51.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test down+unknown '' '' 2026-02-22T22:57:51.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:51.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=down+unknown 2026-02-22T22:57:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:57:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:57:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:57:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:57:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T22:57:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:57:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:57:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T22:57:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T22:57:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22T22:57:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:57:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T22:57:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:57:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801065,"replay_state":"idle"} 2026-02-22T22:57:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:57:47 2026-02-22T22:57:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T22:57:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 3 .mirror.primary.8096762c-9a0b-43fb-82e8-de06ca0535b0.83099494-d1af-4988-84e5-c6ec00c6bef6 (peer_uuids:[]) 2026-02-22T22:57:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 4 .mirror.primary.8096762c-9a0b-43fb-82e8-de06ca0535b0.ba75bdb5-f535-4a16-b906-7dd58f713e5a (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T22:57:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:57:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: down+unknown 2026-02-22T22:57:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:57:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: status not found 2026-02-22T22:57:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:51.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo down+unknown 2026-02-22T22:57:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:57:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: ' 2026-02-22T22:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:57:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update= 2026-02-22T22:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test down+unknown '' 2026-02-22T22:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=down+unknown 2026-02-22T22:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local prev_last_update= 2026-02-22T22:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:57:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:57:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='UNKNOWNUNKNOWNOK1test8096762c-9a0b-43fb-82e8-de06ca0535b0down+unknownstatus not foundcluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801065,"replay_state":"idle"}2026-02-22 22:57:47' 2026-02-22T22:57:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:57:51.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:57:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update= 2026-02-22T22:57:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:57:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=down+unknown 2026-02-22T22:57:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo down+unknown 2026-02-22T22:57:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep down+unknown 2026-02-22T22:57:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:down+unknown 2026-02-22T22:57:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test 2026-02-22T22:57:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T22:57:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T22:57:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.export 2026-02-22T22:57:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:57:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T22:57:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T22:57:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.export 2026-02-22T22:57:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:57:51.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:57:51.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:57:51.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T22:57:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:57:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:57:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:57:51.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:57:51.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:57:51.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:51.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_image_meta cluster1 mirror test key1 value1 2026-02-22T22:57:51.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:51.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:51.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:51.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local key=key1 2026-02-22T22:57:51.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local value=value1 2026-02-22T22:57:51.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 image-meta get mirror/test key1 2026-02-22T22:57:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test value1 = value1 2026-02-22T22:57:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_image_meta cluster1 mirror test key2 value2 2026-02-22T22:57:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local key=key2 2026-02-22T22:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local value=value2 2026-02-22T22:57:51.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 image-meta get mirror/test key2 2026-02-22T22:57:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test value2 = value2 2026-02-22T22:57:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: stop mirror, add image, start mirror and test replay' 2026-02-22T22:57:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T22:57:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T22:57:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 22:57:51 TEST: stop mirror, add image, start mirror and test replay 2026-02-22T22:57:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 22:57:51 TEST: stop mirror, add image, start mirror and test replay 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirrors cluster1 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T22:57:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:0 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T22:57:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:57:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:57:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:57:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:57:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:57:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=49430 2026-02-22T22:57:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 49430 ']' 2026-02-22T22:57:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 49430 2026-02-22T22:57:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:57:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:57:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T22:57:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=49430 '$2 == pid {print; exit 1}' 2026-02-22T22:57:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 49430 1.4 0.0 0 0 ? Rs 22:57 0:00 [rbd-mirror] 2026-02-22T22:57:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:57:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:57:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T22:57:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=49430 '$2 == pid {print; exit 1}' 2026-02-22T22:57:52.598 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T22:57:52.598 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T22:57:52.598 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=49430 '$2 == pid {print; exit 1}' 2026-02-22T22:57:52.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:57:52.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:57:52.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:57:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:57:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:57:52.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:52.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:57:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:52.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:57:52.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:57:52.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:57:52.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:52.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:57:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:52.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:57:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:57:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:57:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:57:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:52.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:57:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:57:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:52.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:57:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:57:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:57:52.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:52.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T22:57:52.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:57:52.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T22:57:52.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:52.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:1 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T22:57:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T22:57:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:57:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:57:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:57:52.680 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:57:52.680 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:57:52.681 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=49474 2026-02-22T22:57:52.681 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 49474 ']' 2026-02-22T22:57:52.681 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 49474 2026-02-22T22:57:52.681 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:57:52.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:57:52.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T22:57:52.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=49474 '$2 == pid {print; exit 1}' 2026-02-22T22:57:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T22:57:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T22:57:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=49474 '$2 == pid {print; exit 1}' 2026-02-22T22:57:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:57:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:57:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:57:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:57:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:57:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:57:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:57:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:57:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:57:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:57:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:57:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:57:52.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:57:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:57:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:57:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T22:57:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:57:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:57:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:57:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:57:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:57:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:57:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:57:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:57:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:57:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:57:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:57:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:57:52.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:57:52.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:57:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:1 2026-02-22T22:57:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:57:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:57:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:57:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:57:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:57:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:57:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:57:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:57:52.773 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:57:52.773 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:57:52.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T22:57:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:57:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T22:57:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T22:57:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T22:57:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:2 2026-02-22T22:57:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:57:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T22:57:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:57:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:57:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:57:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T22:57:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T22:57:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T22:57:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T22:57:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:57:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:57:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:57:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=49533 2026-02-22T22:57:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 49533 ']' 2026-02-22T22:57:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 49533 2026-02-22T22:57:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:57:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:57:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T22:57:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=49533 '$2 == pid {print; exit 1}' 2026-02-22T22:57:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T22:57:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T22:57:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=49533 '$2 == pid {print; exit 1}' 2026-02-22T22:57:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:57:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:57:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:57:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:57:52.843 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:57:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:57:52.853 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:57:52.855 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T22:57:52.855 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:57:52.855 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:57:52.855 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.855 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:57:52.855 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:57:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:57:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:57:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:57:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:57:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:57:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:57:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:57:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:57:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:57:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:57:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:2 2026-02-22T22:57:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T22:57:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:57:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:57:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:57:52.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:57:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T22:57:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:57:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T22:57:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T22:57:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T22:57:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:3 2026-02-22T22:57:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:57:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T22:57:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:57:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:57:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:57:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T22:57:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T22:57:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T22:57:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T22:57:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:57:52.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:57:52.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:57:52.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=49602 2026-02-22T22:57:52.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 49602 ']' 2026-02-22T22:57:52.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 49602 2026-02-22T22:57:52.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:57:52.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:57:52.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T22:57:52.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=49602 '$2 == pid {print; exit 1}' 2026-02-22T22:57:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T22:57:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T22:57:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=49602 '$2 == pid {print; exit 1}' 2026-02-22T22:57:52.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:57:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:57:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:57:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:57:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:57:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:57:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:3 2026-02-22T22:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T22:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:57:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:57:52.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:52.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:52.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:57:52.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:57:52.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:57:52.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:57:52.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:57:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image1=test1 2026-02-22T22:57:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test1 snapshot 2026-02-22T22:57:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test1 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T22:57:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test1 2026-02-22T22:57:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:57:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T22:57:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test1 snapshot 2026-02-22T22:57:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:53.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:53.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T22:57:53.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test1 snapshot 2026-02-22T22:57:53.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T22:57:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test1 2026-02-22T22:57:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test1': 2026-02-22T22:57:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 10e1409285f3 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.10e1409285f3 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 22:57:53 2026 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 22:57:53 2026 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 22:57:53 2026 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:57:53.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T22:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T22:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T22:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:53.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T22:57:53.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:57:53.860 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:57:53.871 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:57:53.871 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 9091 bytes/sec: 36 MiB/s 2026-02-22T22:57:53.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirrors cluster1 2026-02-22T22:57:53.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:53.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:57:53.875 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:53.875 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:0 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:53.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:57:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:1 2026-02-22T22:57:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:57:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:57:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:57:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:57:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:57:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:57:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:57:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:57:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:57:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:57:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:57:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:57:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:57:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:2 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:57:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:3 2026-02-22T22:57:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:57:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:57:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:57:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:57:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:57:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:57:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T22:57:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T22:57:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:57:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:57:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:57:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:57:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T22:57:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:57:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:57:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:57:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:57:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:57:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:57:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:57:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:57:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:57:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:57:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:57:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:57:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:57:54.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:57:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:57:55.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:57:55.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:57:55.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:57:55.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:57:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:57:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:57:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:57:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:57:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:57:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:57:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:57:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:57:55.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:57:55.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:57:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:57:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T22:57:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:57:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:57:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:57:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:57:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T22:57:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:57:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:57:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:57:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:57:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:57:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:57:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:57:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:57:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:57:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:57:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T22:57:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:57:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:57:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:57:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:57:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:57:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T22:57:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T22:57:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:57:55.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:57:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:57:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:57:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T22:57:55.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:57:55.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:57:55.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test1 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T22:57:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:57:55.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:57:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:57:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:57:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test1 2026-02-22T22:57:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:57:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:57:55.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:57:55.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:57:55.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T22:57:56.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:57:56.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:57:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:57:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:57:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:57:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:57:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:57:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:57:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:57:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:57:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:57:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:57:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:57:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:57:56.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:57:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:57:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:57:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:57:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:57:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:57:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:57:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:57:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test1 2026-02-22T22:57:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:57:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:57:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:57:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test1 2026-02-22T22:57:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test1 2026-02-22T22:57:56.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 6 2026-02-22T22:57:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test1 2026-02-22T22:57:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:57:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test1.status 2026-02-22T22:57:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test1.status 2026-02-22T22:57:56.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:57:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test1.status 2026-02-22T22:57:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test1.status 2026-02-22T22:57:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test1.status 2026-02-22T22:57:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:57:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.status 2026-02-22T22:57:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test1 /tmp/tmp.VQMrvUqClS/cluster2-mirror-test1.status 2026-02-22T22:57:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test1.status 2026-02-22T22:57:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test1 --format xml 2026-02-22T22:57:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:57:56.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:57:56.900 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=6 2026-02-22T22:57:56.900 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T22:57:56.900 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:57:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T22:57:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.status 2026-02-22T22:57:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:57.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.status 2026-02-22T22:57:57.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-02-22T22:57:57.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:57:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:57:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=5 2026-02-22T22:57:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 6 = 5 2026-02-22T22:57:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:57:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T22:57:57.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.status 2026-02-22T22:57:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.status 2026-02-22T22:57:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-02-22T22:57:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:57:57.565 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:57:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=5 2026-02-22T22:57:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 6 = 5 2026-02-22T22:57:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:57:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T22:57:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.status 2026-02-22T22:57:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:58.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.status 2026-02-22T22:57:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-02-22T22:57:58.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:57:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:57:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=6 2026-02-22T22:57:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 6 = 6 2026-02-22T22:57:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test1 2026-02-22T22:57:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:57:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T22:57:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:57:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:57:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:57:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:57:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:57:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T22:57:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:57:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:57:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:57:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:57:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:57:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:57:59.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:57:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:57:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:57:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801073,"replay_state":"idle"} 2026-02-22T22:57:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:57:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:57:55' 2026-02-22T22:57:59.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:57:59.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:57:59.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:57:59.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801073,"replay_state":"idle"} 2026-02-22T22:57:59.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:57:59.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:57:59.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:57:59.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:57:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:57:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:57:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:57:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:57:59.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:57:55' 2026-02-22T22:57:59.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:57:55' 2026-02-22T22:57:59.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:57:59.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:57:55' 2026-02-22T22:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:57:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:57:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801068,"remote_snapshot_timestamp":1771801068,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:57:57test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801073,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:57:55' 2026-02-22T22:57:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:57:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:57:59.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:57:55' 2026-02-22T22:57:59.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:57:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:57:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:57:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:57:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:57:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:57:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z '' ']' 2026-02-22T22:57:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:57:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:57:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:57:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:57:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=down+unknown 2026-02-22T22:57:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:57:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:57:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:57:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:58:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=down+unknown 2026-02-22T22:58:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:58:00.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:58:00.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:58:00.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T22:58:00.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T22:58:00.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22T22:58:00.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:58:00.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T22:58:00.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:00.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801073,"replay_state":"idle"} 2026-02-22T22:58:00.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:57:55 2026-02-22T22:58:00.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T22:58:00.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 5 .mirror.primary.6fe99b27-5251-4912-94c9-15fe1f412bc8.3c0a5ca6-e221-4a53-9bf4-e803245ba1b2 (peer_uuids:[]) 2026-02-22T22:58:00.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 6 .mirror.primary.6fe99b27-5251-4912-94c9-15fe1f412bc8.7a29e20e-783d-4c4f-b9ab-a6099ea340f4 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T22:58:00.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:58:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: down+unknown 2026-02-22T22:58:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: status not found 2026-02-22T22:58:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo down+unknown 2026-02-22T22:58:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: ' 2026-02-22T22:58:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update= 2026-02-22T22:58:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:58:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:00.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=down+unknown 2026-02-22T22:58:00.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local prev_last_update= 2026-02-22T22:58:00.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:00.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:00.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='UNKNOWNUNKNOWNOK2test8096762c-9a0b-43fb-82e8-de06ca0535b0down+unknownstatus not foundcluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801068,"remote_snapshot_timestamp":1771801068,"replay_state":"idle"}2026-02-22 22:57:57test16fe99b27-5251-4912-94c9-15fe1f412bc8down+unknownstatus not foundcluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801073,"replay_state":"idle"}2026-02-22 22:57:55' 2026-02-22T22:58:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:00.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:58:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update= 2026-02-22T22:58:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:58:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=down+unknown 2026-02-22T22:58:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo down+unknown 2026-02-22T22:58:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep down+unknown 2026-02-22T22:58:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:down+unknown 2026-02-22T22:58:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test1 2026-02-22T22:58:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T22:58:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:58:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:58:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:58:00.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test1.export 2026-02-22T22:58:00.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:58:00.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:58:00.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:58:00.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.export 2026-02-22T22:58:00.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test1.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.export 2026-02-22T22:58:00.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.VQMrvUqClS/cluster2-mirror-test1.export 2026-02-22T22:58:00.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:58:00.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.export 2026-02-22T22:58:00.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:58:00.729 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test1.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.export 2026-02-22T22:58:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test1.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test1.export 2026-02-22T22:58:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: test the first image is replaying after restart' 2026-02-22T22:58:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T22:58:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T22:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 22:58:00 TEST: test the first image is replaying after restart 2026-02-22T22:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 22:58:00 TEST: test the first image is replaying after restart 2026-02-22T22:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test 100 2026-02-22T22:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T22:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T22:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:58:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T22:58:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:58:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:58:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:58:00.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T22:58:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:58:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:58:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:01.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:58:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:01.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:01.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:01.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:01.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:01.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:01.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:01.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:01.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:01.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:01.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:01.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:01.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:01.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:01.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:01.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:01.930 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T22:58:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:58:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:58:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:58:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T22:58:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:01.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T22:58:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 7 2026-02-22T22:58:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T22:58:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:58:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T22:58:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T22:58:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T22:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T22:58:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:58:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T22:58:03.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:58:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:58:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=7 2026-02-22T22:58:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T22:58:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:58:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T22:58:04.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:58:04.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:04.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:04.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:04.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:58:04.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T22:58:04.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:58:04.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:58:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=7 2026-02-22T22:58:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 7 = 7 2026-02-22T22:58:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:58:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801068,"remote_snapshot_timestamp":1771801068,"replay_state":"idle"} 2026-02-22T22:58:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:57:57' 2026-02-22T22:58:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801068,"remote_snapshot_timestamp":1771801068,"replay_state":"idle"} 2026-02-22T22:58:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:57:57' 2026-02-22T22:58:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:57:57' 2026-02-22T22:58:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:57:57' 2026-02-22T22:58:05.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:05.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:05.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801068,"remote_snapshot_timestamp":1771801068,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:57:57test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801073,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:57:55' 2026-02-22T22:58:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:57:57' 2026-02-22T22:58:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test 2026-02-22T22:58:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T22:58:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T22:58:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.export 2026-02-22T22:58:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:58:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T22:58:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T22:58:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.export 2026-02-22T22:58:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:58:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:58:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:58:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T22:58:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:58:05.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:58:05.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:58:05.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:58:05.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:58:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z '' ']' 2026-02-22T22:58:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: stop/start/restart mirror via admin socket' 2026-02-22T22:58:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T22:58:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T22:58:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 22:58:05 TEST: stop/start/restart mirror via admin socket 2026-02-22T22:58:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 22:58:05 TEST: stop/start/restart mirror via admin socket 2026-02-22T22:58:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror stop 2026-02-22T22:58:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:05.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:05.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror stop 2026-02-22T22:58:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:05.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:05.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:05.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:05.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:05.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop 2026-02-22T22:58:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror stop 2026-02-22T22:58:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop 2026-02-22T22:58:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror stop 2026-02-22T22:58:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:05.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:05.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:05.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:05.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop 2026-02-22T22:58:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror stop 2026-02-22T22:58:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:05.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:05.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:05.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:05.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:05.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop 2026-02-22T22:58:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T22:58:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T22:58:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:58:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T22:58:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:58:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:06.735 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:06.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:06.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:06.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:58:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:58:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:58:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T22:58:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:58:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:58:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:58:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:58:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:58:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:58:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T22:58:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:07.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:07.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:07.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T22:58:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:58:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: stopped 2026-02-22T22:58:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:05' 2026-02-22T22:58:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:58:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T22:58:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: stopped 2026-02-22T22:58:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:08.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:08.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:08.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:05' 2026-02-22T22:58:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 22:58:05' 2026-02-22T22:58:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:05' 2026-02-22T22:58:08.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:08.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:08.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+stoppedstopped43284333mirror.1trial0822026-02-22 22:58:05test16fe99b27-5251-4912-94c9-15fe1f412bc8up+stoppedstopped43224326mirror.0trial0822026-02-22 22:58:05' 2026-02-22T22:58:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:05' 2026-02-22T22:58:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T22:58:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T22:58:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T22:58:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:09.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T22:58:09.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:09.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:58:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:58:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:58:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:58:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: stopped 2026-02-22T22:58:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:58:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:05' 2026-02-22T22:58:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:58:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T22:58:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: stopped 2026-02-22T22:58:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:10.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:10.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:58:10.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:10.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:05' 2026-02-22T22:58:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 22:58:05' 2026-02-22T22:58:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:05' 2026-02-22T22:58:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+stoppedstopped43284333mirror.1trial0822026-02-22 22:58:05test16fe99b27-5251-4912-94c9-15fe1f412bc8up+stoppedstopped43224326mirror.0trial0822026-02-22 22:58:05' 2026-02-22T22:58:10.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:10.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:58:10.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:05' 2026-02-22T22:58:10.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:58:10.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T22:58:10.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:10.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T22:58:10.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror start 2026-02-22T22:58:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror start 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start 2026-02-22T22:58:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror start 2026-02-22T22:58:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:10.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:10.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:10.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:10.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:10.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:10.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:10.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:10.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:10.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:10.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:10.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:10.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:10.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start 2026-02-22T22:58:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror start 2026-02-22T22:58:10.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:10.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror start 2026-02-22T22:58:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror start 2026-02-22T22:58:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:10.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:10.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror start 2026-02-22T22:58:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:58:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:58:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:58:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:11.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:11.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:11.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:11.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:58:12.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:12.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:12.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:58:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:12.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:12.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:12.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:58:12.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:12.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:12.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:12.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:12.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:12.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:58:12.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:12.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:12.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:58:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:10' 2026-02-22T22:58:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:13.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:13.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:10' 2026-02-22T22:58:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:58:10' 2026-02-22T22:58:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:10' 2026-02-22T22:58:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:13.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:10test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:58:10' 2026-02-22T22:58:13.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:13.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:13.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:10' 2026-02-22T22:58:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:13.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:13.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:13.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:13.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T22:58:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T22:58:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:58:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:58:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:58:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"} 2026-02-22T22:58:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:58:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:10' 2026-02-22T22:58:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"} 2026-02-22T22:58:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:14.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:14.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:14.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:58:14.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:14.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:10' 2026-02-22T22:58:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:58:10' 2026-02-22T22:58:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:10' 2026-02-22T22:58:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:14.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:14.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:10test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:58:10' 2026-02-22T22:58:14.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:14.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:58:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:10' 2026-02-22T22:58:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:58:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror restart 2026-02-22T22:58:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror restart 2026-02-22T22:58:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart 2026-02-22T22:58:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror restart 2026-02-22T22:58:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:14.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:14.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:14.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:14.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart 2026-02-22T22:58:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror restart 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart 2026-02-22T22:58:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror restart 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:14.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:14.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:14.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:14.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart 2026-02-22T22:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:58:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:15.873 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:15.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:15.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:15.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:15.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:15.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:15.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:15.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T22:58:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:17.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:17.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:17.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:17.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:58:17.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:58:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:58:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:14' 2026-02-22T22:58:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:18.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:18.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:18.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:14' 2026-02-22T22:58:18.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:58:14' 2026-02-22T22:58:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:14' 2026-02-22T22:58:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:18.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:14test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:58:14' 2026-02-22T22:58:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:18.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:14' 2026-02-22T22:58:18.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T22:58:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T22:58:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:58:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:58:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:58:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"} 2026-02-22T22:58:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:58:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:14' 2026-02-22T22:58:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"} 2026-02-22T22:58:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:58:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:14' 2026-02-22T22:58:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:58:14' 2026-02-22T22:58:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:14' 2026-02-22T22:58:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:19.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:14test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:58:14' 2026-02-22T22:58:19.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:19.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:58:19.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:14' 2026-02-22T22:58:19.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:58:19.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:19.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:19.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror stop 2026-02-22T22:58:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:19.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror stop 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop 2026-02-22T22:58:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror stop 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:19.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:19.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:19.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop 2026-02-22T22:58:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror stop 2026-02-22T22:58:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:19.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:19.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:19.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:19.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop 2026-02-22T22:58:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror stop 2026-02-22T22:58:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop 2026-02-22T22:58:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T22:58:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T22:58:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:58:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T22:58:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:20.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:20.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:20.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:20.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:20.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:58:20.565 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:58:20.565 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:20.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:58:20.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T22:58:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:58:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:58:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:21.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:58:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:21.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:58:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:58:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:58:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:58:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T22:58:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T22:58:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:58:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: stopped 2026-02-22T22:58:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:19' 2026-02-22T22:58:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:58:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T22:58:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: stopped 2026-02-22T22:58:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:22.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:22.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:22.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:22.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:22.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:19' 2026-02-22T22:58:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 22:58:19' 2026-02-22T22:58:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:19' 2026-02-22T22:58:22.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:22.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:22.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+stoppedstopped43284333mirror.1trial0822026-02-22 22:58:19test16fe99b27-5251-4912-94c9-15fe1f412bc8up+stoppedstopped43224326mirror.0trial0822026-02-22 22:58:19' 2026-02-22T22:58:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:19' 2026-02-22T22:58:22.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:22.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T22:58:22.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T22:58:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T22:58:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T22:58:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:23.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:58:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:58:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:58:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:58:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: stopped 2026-02-22T22:58:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:58:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:19' 2026-02-22T22:58:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:58:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T22:58:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: stopped 2026-02-22T22:58:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:58:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:19' 2026-02-22T22:58:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 22:58:19' 2026-02-22T22:58:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:19' 2026-02-22T22:58:23.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:23.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+stoppedstopped43284333mirror.1trial0822026-02-22 22:58:19test16fe99b27-5251-4912-94c9-15fe1f412bc8up+stoppedstopped43224326mirror.0trial0822026-02-22 22:58:19' 2026-02-22T22:58:23.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:58:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:19' 2026-02-22T22:58:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:58:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T22:58:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T22:58:23.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:23.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:23.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:23.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror restart 2026-02-22T22:58:23.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:23.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:23.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror restart 2026-02-22T22:58:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:23.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:23.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:23.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:23.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:23.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart 2026-02-22T22:58:23.849 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror restart 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:23.850 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart 2026-02-22T22:58:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror restart 2026-02-22T22:58:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:23.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:23.914 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:23.914 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:23.914 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart 2026-02-22T22:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror restart 2026-02-22T22:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:58:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:25.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:25.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:25.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:25.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:25.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:25.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:25.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:25.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:25.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T22:58:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:58:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:26.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:26.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:58:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:58:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:27.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:58:27.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:27.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:27.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:27.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:27.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:27.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:27.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:27.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:27.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:23' 2026-02-22T22:58:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:27.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:27.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:27.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:27.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:27.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:27.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:23' 2026-02-22T22:58:27.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:58:23' 2026-02-22T22:58:27.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:27.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:23' 2026-02-22T22:58:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:27.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:23test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:58:23' 2026-02-22T22:58:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:23' 2026-02-22T22:58:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T22:58:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:27.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T22:58:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:58:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:58:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:58:28.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:28.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"} 2026-02-22T22:58:28.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:58:28.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:23' 2026-02-22T22:58:28.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:28.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:28.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:28.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"} 2026-02-22T22:58:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:58:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:23' 2026-02-22T22:58:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:58:23' 2026-02-22T22:58:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:23' 2026-02-22T22:58:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:23test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:58:23' 2026-02-22T22:58:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:58:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:23' 2026-02-22T22:58:28.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:58:28.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:28.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:28.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:28.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:28.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:28.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:28.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:28.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:28.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:28.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:28.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:28.488 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:28.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:28.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T22:58:28.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:28.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:28.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:28.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T22:58:28.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:28.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:28.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:28.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:58:28.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:28.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:28.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:29.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T22:58:29.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:29.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:29.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:29.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:58:29.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:29.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:29.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:29.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:29.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:29.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:29.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:29.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:29.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:29.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:29.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:29.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:29.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:29.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:29.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:29.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:29.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:29.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:29.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:29.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:29.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:29.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:29.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:29.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:29.677 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:58:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:58:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:29.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:58:29.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:29.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T22:58:29.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:29.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T22:58:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:58:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T22:58:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:58:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:58:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:58:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:58:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T22:58:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T22:58:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:31.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:31.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:31.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:58:31.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: stopped 2026-02-22T22:58:31.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:31.822 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:28' 2026-02-22T22:58:31.822 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:58:31.822 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T22:58:31.822 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: stopped 2026-02-22T22:58:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:31.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:31.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:31.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:28' 2026-02-22T22:58:31.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 22:58:28' 2026-02-22T22:58:31.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:31.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:31.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:31.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:31.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:28' 2026-02-22T22:58:31.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:31.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:31.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+stoppedstopped43284333mirror.1trial0822026-02-22 22:58:28test16fe99b27-5251-4912-94c9-15fe1f412bc8up+stoppedstopped43224326mirror.0trial0822026-02-22 22:58:28' 2026-02-22T22:58:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:31.860 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:28' 2026-02-22T22:58:31.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T22:58:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T22:58:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T22:58:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:32.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T22:58:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:32.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:58:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:58:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:58:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:58:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: stopped 2026-02-22T22:58:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:58:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:28' 2026-02-22T22:58:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:58:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T22:58:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: stopped 2026-02-22T22:58:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:58:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:32.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:28' 2026-02-22T22:58:32.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 22:58:28' 2026-02-22T22:58:32.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:32.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:32.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:32.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:32.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:28' 2026-02-22T22:58:32.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:32.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:32.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:32.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+stoppedstopped43284333mirror.1trial0822026-02-22 22:58:28test16fe99b27-5251-4912-94c9-15fe1f412bc8up+stoppedstopped43224326mirror.0trial0822026-02-22 22:58:28' 2026-02-22T22:58:32.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:32.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:58:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:28' 2026-02-22T22:58:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:58:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T22:58:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T22:58:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-22T22:58:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_instance=cluster1 2026-02-22T22:58:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance=cluster1 2026-02-22T22:58:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loop_instance 2026-02-22T22:58:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:58:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:32.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-22T22:58:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror start mirror/test1 2026-02-22T22:58:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror/test 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:33.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror/test 2026-02-22T22:58:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:58:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:34.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:34.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:34.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:58:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:58:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:35.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:35.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:35.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:33' 2026-02-22T22:58:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:33' 2026-02-22T22:58:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:58:33' 2026-02-22T22:58:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:33' 2026-02-22T22:58:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:35.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:33test16fe99b27-5251-4912-94c9-15fe1f412bc8up+stoppedstopped43224326mirror.0trial0822026-02-22 22:58:28' 2026-02-22T22:58:35.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:33' 2026-02-22T22:58:35.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:35.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror start mirror cluster2-DNE 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror cluster2-DNE 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror cluster2-DNE 2026-02-22T22:58:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror cluster2-DNE 2026-02-22T22:58:35.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:35.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:35.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:35.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:35.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:35.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror cluster2-DNE 2026-02-22T22:58:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror start mirror cluster2-DNE 2026-02-22T22:58:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:35.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:35.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:35.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:35.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror start mirror cluster2-DNE 2026-02-22T22:58:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror start mirror cluster2-DNE 2026-02-22T22:58:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:35.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:35.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:35.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:35.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:35.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:35.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:35.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:35.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror start mirror cluster2-DNE 2026-02-22T22:58:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T22:58:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:35.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:35.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:35.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:35.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:35.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:35.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:35.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:36.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:36.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:36.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:36.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:58:36.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:36.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:36.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:36.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:58:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T22:58:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T22:58:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:58:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:58:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:58:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"} 2026-02-22T22:58:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:58:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:35' 2026-02-22T22:58:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:37.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"} 2026-02-22T22:58:37.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:37.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:37.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:37.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:37.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:58:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:35' 2026-02-22T22:58:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:58:35' 2026-02-22T22:58:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:35' 2026-02-22T22:58:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:37.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:37.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:37.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:33test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:58:35' 2026-02-22T22:58:37.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:37.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:58:37.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:35' 2026-02-22T22:58:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:58:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemons cluster1 rbd mirror restart mirror/test 2026-02-22T22:58:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_instance=cluster1 2026-02-22T22:58:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance=cluster1 2026-02-22T22:58:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loop_instance 2026-02-22T22:58:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:58:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror/test 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror/test 2026-02-22T22:58:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror restart mirror/test1 2026-02-22T22:58:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror/test 2026-02-22T22:58:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:37.686 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:37.686 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:37.686 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:37.693 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:37.693 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:37.693 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:37.693 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror/test 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:37.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:58:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:38.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:38.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:58:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:39.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:39.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:39.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:39.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:39.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:39.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:37' 2026-02-22T22:58:39.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:39.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:39.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:39.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:39.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:39.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:39.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:37' 2026-02-22T22:58:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:58:37' 2026-02-22T22:58:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:37' 2026-02-22T22:58:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:39.933 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:37test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:58:35' 2026-02-22T22:58:39.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:39.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:37' 2026-02-22T22:58:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:39.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:39.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:39.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:39.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:39.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:58:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:58:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:58:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:41.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:41.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:41.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:41.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:41.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:41.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:58:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:58:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:58:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:42.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:42.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:42.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:42.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:42.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:42.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:42.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:42.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:42.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:42.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:42.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:42.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:42.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:42.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:42.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:42.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop mirror cluster2-DNE 2026-02-22T22:58:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T22:58:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T22:58:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:58:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T22:58:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:43.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:58:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:43.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:43.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:43.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:43.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:43.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:43.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:43.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:43.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:43.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:43.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:43.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:43.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:43.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:43.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:43.722 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:43.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:58:43.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:58:43.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:58:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T22:58:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T22:58:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:58:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T22:58:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:58:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:44.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:44.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:44.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:44.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:58:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T22:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T22:58:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:45.823 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:58:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: stopped 2026-02-22T22:58:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:42' 2026-02-22T22:58:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:58:45.825 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T22:58:45.825 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:45.825 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: stopped 2026-02-22T22:58:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:45.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:45.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:42' 2026-02-22T22:58:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 22:58:42' 2026-02-22T22:58:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:42' 2026-02-22T22:58:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:45.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:45.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:45.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+stoppedstopped43284333mirror.1trial0822026-02-22 22:58:42test16fe99b27-5251-4912-94c9-15fe1f412bc8up+stoppedstopped43224326mirror.0trial0822026-02-22 22:58:42' 2026-02-22T22:58:45.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:45.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:45.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:42' 2026-02-22T22:58:45.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:45.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T22:58:45.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:45.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T22:58:45.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T22:58:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:45.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:45.870 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:45.870 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:46.870 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T22:58:46.870 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:46.870 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:46.870 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:46.870 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:46.871 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:46.871 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:46.871 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:46.871 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:46.871 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:58:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:58:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:58:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:58:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: stopped 2026-02-22T22:58:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:58:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:42' 2026-02-22T22:58:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:58:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T22:58:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: stopped 2026-02-22T22:58:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:58:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:42' 2026-02-22T22:58:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 22:58:42' 2026-02-22T22:58:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:58:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:42' 2026-02-22T22:58:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+stoppedstopped43284333mirror.1trial0822026-02-22 22:58:42test16fe99b27-5251-4912-94c9-15fe1f412bc8up+stoppedstopped43224326mirror.0trial0822026-02-22 22:58:42' 2026-02-22T22:58:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:58:46.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:42' 2026-02-22T22:58:46.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:58:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T22:58:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:58:46.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T22:58:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:58:46.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:46.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:46.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:46.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:46.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:46.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:46.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:47.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:47.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:47.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:47.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:47.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:47.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:47.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:47.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:47.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:47.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:47.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart mirror cluster2-DNE 2026-02-22T22:58:47.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:58:47.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:47.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:58:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:47.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:58:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:48.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:58:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:48.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:48.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:58:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:no valid command found; 1 closest matches: 2026-02-22T22:58:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status mirror/test1 2026-02-22T22:58:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:58:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:48.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:48.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:58:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:48.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T22:58:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T22:58:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:58:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:58:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:49.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:49.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:49.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:49.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:58:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:58:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:58:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:58:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:47' 2026-02-22T22:58:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:50.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"} 2026-02-22T22:58:50.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:50.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:50.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:50.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:50.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:50.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:50.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:50.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:50.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:47' 2026-02-22T22:58:50.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:58:47' 2026-02-22T22:58:50.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:50.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:50.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:50.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:50.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:47' 2026-02-22T22:58:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:50.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:47test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:58:47' 2026-02-22T22:58:50.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:50.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:58:50.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:47' 2026-02-22T22:58:50.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:58:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T22:58:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:58:51.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test1: 2026-02-22T22:58:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 6fe99b27-5251-4912-94c9-15fe1f412bc8 2026-02-22T22:58:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"} 2026-02-22T22:58:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:58:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:47' 2026-02-22T22:58:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:51.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"} 2026-02-22T22:58:51.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:51.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:51.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:58:51.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:51.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:47' 2026-02-22T22:58:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:58:47' 2026-02-22T22:58:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:47' 2026-02-22T22:58:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:51.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801082,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:47test16fe99b27-5251-4912-94c9-15fe1f412bc8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801076,"remote_snapshot_timestamp":1771801076,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:58:47' 2026-02-22T22:58:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:58:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:47' 2026-02-22T22:58:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:58:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ flush cluster1 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool= 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image= 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemons cluster1 rbd mirror flush 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_instance=cluster1 2026-02-22T22:58:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance=cluster1 2026-02-22T22:58:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loop_instance 2026-02-22T22:58:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:58:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror flush 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush 2026-02-22T22:58:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ all_admin_daemons cluster1 rbd mirror status 2026-02-22T22:58:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror status 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status 2026-02-22T22:58:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:{ 2026-02-22T22:58:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool_replayers": [ 2026-02-22T22:58:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "peer": "uuid: 70a926a6-5554-4bd7-a6ec-e3f48f4efa7b cluster: cluster2-DNE client: client.mirror", 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool": "mirror", 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "instance_id": "4326", 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "stopped (manual)", 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader_instance_id": "4326", 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader": true, 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "instances": [ 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "4326", 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "4333", 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "4338", 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "4349" 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ], 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.0.50406.cluster1.94854416644456.asok", 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50406.cluster2-DNE.94854416963944.asok", 2026-02-22T22:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "sync_throttler": { 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 5, 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "deletion_throttler": { 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 1, 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [ 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "mirror/test1", 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ], 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_deleter": { 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_deleter_status": { 2026-02-22T22:58:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "delete_images_queue": [], 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "failed_deletes_queue": [] 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "namespaces": [ 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns1", 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [], 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_deleter": { 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_deleter_status": { 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "delete_images_queue": [], 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "failed_deletes_queue": [] 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns2", 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [], 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_deleter": { 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_deleter_status": { 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "delete_images_queue": [], 2026-02-22T22:58:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "failed_deletes_queue": [] 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "peer": "uuid: a5da3887-dec5-4279-9da0-8412a090cb5c cluster: cluster2-DNE client: client.mirror", 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool": "mirror_parent", 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "instance_id": "4342", 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "stopped (manual)", 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader_instance_id": "4342", 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader": true, 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "instances": [ 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "4342", 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "4348", 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "4352", 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "4354" 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ], 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.0.50406.cluster1.94854420076904.asok", 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50406.cluster2-DNE.94854422190440.asok", 2026-02-22T22:58:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "sync_throttler": { 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 5, 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "deletion_throttler": { 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 1, 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [], 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_deleter": { 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_deleter_status": { 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "delete_images_queue": [], 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "failed_deletes_queue": [] 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "namespaces": [ 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns1", 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [], 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_deleter": { 2026-02-22T22:58:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_deleter_status": { 2026-02-22T22:58:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "delete_images_queue": [], 2026-02-22T22:58:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "failed_deletes_queue": [] 2026-02-22T22:58:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:} 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror status 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status 2026-02-22T22:58:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:{ 2026-02-22T22:58:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool_replayers": [ 2026-02-22T22:58:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "peer": "uuid: 70a926a6-5554-4bd7-a6ec-e3f48f4efa7b cluster: cluster2-DNE client: client.mirror", 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool": "mirror", 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "instance_id": "4333", 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "stopped (manual)", 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader_instance_id": "4326", 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader": false, 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.1.50445.cluster1.94192166320488.asok", 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50445.cluster2-DNE.94192179173736.asok", 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "sync_throttler": { 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 5, 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "deletion_throttler": { 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 1, 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [ 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "mirror/test", 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ], 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "namespaces": [ 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns1", 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [] 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns2", 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [] 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "peer": "uuid: a5da3887-dec5-4279-9da0-8412a090cb5c cluster: cluster2-DNE client: client.mirror", 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool": "mirror_parent", 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "instance_id": "4348", 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "stopped (manual)", 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader_instance_id": "4342", 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader": false, 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.1.50445.cluster1.94192169793896.asok", 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50445.cluster2-DNE.94192171727208.asok", 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "sync_throttler": { 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 5, 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "deletion_throttler": { 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 1, 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [], 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "namespaces": [ 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns1", 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [] 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:} 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:2 rbd mirror status 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:58:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:58:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:58:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:58:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status 2026-02-22T22:58:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:{ 2026-02-22T22:58:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool_replayers": [ 2026-02-22T22:58:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "peer": "uuid: 70a926a6-5554-4bd7-a6ec-e3f48f4efa7b cluster: cluster2-DNE client: client.mirror", 2026-02-22T22:58:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool": "mirror", 2026-02-22T22:58:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "instance_id": "4338", 2026-02-22T22:58:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "stopped (manual)", 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader_instance_id": "4326", 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader": false, 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.2.50510.cluster1.94258214828392.asok", 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50510.cluster2-DNE.94258227706216.asok", 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "sync_throttler": { 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 5, 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "deletion_throttler": { 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 1, 2026-02-22T22:58:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [], 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "namespaces": [ 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns1", 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [] 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns2", 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [] 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "peer": "uuid: a5da3887-dec5-4279-9da0-8412a090cb5c cluster: cluster2-DNE client: client.mirror", 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool": "mirror_parent", 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "instance_id": "4352", 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "stopped (manual)", 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader_instance_id": "4342", 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader": false, 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.2.50510.cluster1.94258218375528.asok", 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50510.cluster2-DNE.94258220210536.asok", 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "sync_throttler": { 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 5, 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "deletion_throttler": { 2026-02-22T22:58:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 1, 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [], 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "namespaces": [ 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns1", 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [] 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:} 2026-02-22T22:58:51.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:3 rbd mirror status 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:58:51.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:58:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:58:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status 2026-02-22T22:58:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:{ 2026-02-22T22:58:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool_replayers": [ 2026-02-22T22:58:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "peer": "uuid: 70a926a6-5554-4bd7-a6ec-e3f48f4efa7b cluster: cluster2-DNE client: client.mirror", 2026-02-22T22:58:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool": "mirror", 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "instance_id": "4349", 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "stopped (manual)", 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader_instance_id": "4326", 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader": false, 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.3.50572.cluster1.94739973667176.asok", 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50572.cluster2-DNE.94739986585960.asok", 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "sync_throttler": { 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 5, 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "deletion_throttler": { 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 1, 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [], 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "namespaces": [ 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns1", 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [] 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns2", 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [] 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "peer": "uuid: a5da3887-dec5-4279-9da0-8412a090cb5c cluster: cluster2-DNE client: client.mirror", 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "pool": "mirror_parent", 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "instance_id": "4354", 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "stopped (manual)", 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader_instance_id": "4342", 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "leader": false, 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.3.50572.cluster1.94739977189736.asok", 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50572.cluster2-DNE.94739979073896.asok", 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "sync_throttler": { 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 5, 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "deletion_throttler": { 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "max_parallel_requests": 1, 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "running_requests": 0, 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "waiting_requests": 0 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: }, 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [], 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "namespaces": [ 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: { 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "ns1", 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "image_replayers": [] 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: } 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: ] 2026-02-22T22:58:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:} 2026-02-22T22:58:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test1 2026-02-22T22:58:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:58:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:58:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test1 2026-02-22T22:58:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test1 2026-02-22T22:58:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test1 2026-02-22T22:58:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test1 2026-02-22T22:58:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:58:51.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:51.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: test image rename' 2026-02-22T22:58:51.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T22:58:51.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T22:58:51.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 22:58:51 TEST: test image rename 2026-02-22T22:58:51.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 22:58:51 TEST: test image rename 2026-02-22T22:58:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ new_name=test_RENAMED 2026-02-22T22:58:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_image cluster2 mirror test test_RENAMED 2026-02-22T22:58:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_name=test_RENAMED 2026-02-22T22:58:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rename mirror/test mirror/test_RENAMED 2026-02-22T22:58:52.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:58:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test_RENAMED 2026-02-22T22:58:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_RENAMED 2026-02-22T22:58:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 8 2026-02-22T22:58:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_RENAMED 2026-02-22T22:58:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T22:58:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T22:58:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:54.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:54.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:54.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:54.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:54.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:54.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:54.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T22:58:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:58:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:55.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:55.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:55.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying '' '' 2026-02-22T22:58:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_RENAMED 2026-02-22T22:58:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_RENAMED: 2026-02-22T22:58:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801132,"replay_state":"syncing","syncing_percent":100,"syncing_snapshot_timestamp":1771801132} 2026-02-22T22:58:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:53' 2026-02-22T22:58:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801132,"replay_state":"syncing","syncing_percent":100,"syncing_snapshot_timestamp":1771801132} 2026-02-22T22:58:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:53' 2026-02-22T22:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_RENAMED up+replaying '2026-02-22 22:58:53' 2026-02-22T22:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:53' 2026-02-22T22:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:56.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:56.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_RENAMED8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801082,"remote_snapshot_timestamp":1771801132,"replay_state":"syncing","syncing_percent":100,"syncing_snapshot_timestamp":1771801132}43284333mirror.1trial0822026-02-22 22:58:53' 2026-02-22T22:58:56.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:56.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/last_update' 2026-02-22T22:58:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:53' 2026-02-22T22:58:56.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/state' 2026-02-22T22:58:56.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:56.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:56.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:56.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_instance=cluster1 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance=cluster1 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loop_instance 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:58:56.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:56.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T22:58:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:58:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:56.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:56.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:56.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:56.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:56.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:{ 2026-02-22T22:58:56.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "name": "mirror/test_RENAMED", 2026-02-22T22:58:56.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:56.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:} 2026-02-22T22:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemons cluster1 rbd mirror restart mirror/test_RENAMED 2026-02-22T22:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_instance=cluster1 2026-02-22T22:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance=cluster1 2026-02-22T22:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loop_instance 2026-02-22T22:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:58:56.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:56.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:58:56.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:56.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror/test_RENAMED 2026-02-22T22:58:56.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:56.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:56.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:56.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:56.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror/test_RENAMED 2026-02-22T22:58:56.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T22:58:56.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror restart : restart rbd mirror 2026-02-22T22:58:56.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:56.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:56.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror/test_RENAMED 2026-02-22T22:58:56.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T22:58:56.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:58:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T22:58:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T22:58:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror/test_RENAMED 2026-02-22T22:58:56.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_RENAMED 2026-02-22T22:58:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T22:58:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:56.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:56.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:56.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T22:58:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:58:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:58:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:58:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:58:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:58:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:58:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:58:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:58:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:58:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:58:57.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:58:57.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:57.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:58:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:58:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:58:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:58:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:58:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:58:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T22:58:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:58:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:58:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:58:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:58:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:58:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:58:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:58:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:58:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:58:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:58:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:58:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:58:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:58:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:58:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:58:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:58:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:58:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:58:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:58:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:58:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:58:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:58:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-02-22T22:58:57.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:58:57.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:58:57.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:58:57.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:58:57.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:58:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:58:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:58:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying 2026-02-22T22:58:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:58:58.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying '' '' 2026-02-22T22:58:58.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:58.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:58.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:58.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:58.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:58:58.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:58:58.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:58.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:58.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_RENAMED 2026-02-22T22:58:58.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_RENAMED: 2026-02-22T22:58:58.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 8096762c-9a0b-43fb-82e8-de06ca0535b0 2026-02-22T22:58:58.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:58:58.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801132,"remote_snapshot_timestamp":1771801132,"replay_state":"idle"} 2026-02-22T22:58:58.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T22:58:58.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:58:56' 2026-02-22T22:58:58.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:58:58.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:58:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:58:58.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801132,"remote_snapshot_timestamp":1771801132,"replay_state":"idle"} 2026-02-22T22:58:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:58:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:58:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:58:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T22:58:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:58:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:58:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:56' 2026-02-22T22:58:58.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_RENAMED up+replaying '2026-02-22 22:58:56' 2026-02-22T22:58:58.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:58.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:58.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:58.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:58:58.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:58:56' 2026-02-22T22:58:58.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:58:58.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:58:58.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:58:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_RENAMED8096762c-9a0b-43fb-82e8-de06ca0535b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801132,"remote_snapshot_timestamp":1771801132,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 22:58:56' 2026-02-22T22:58:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:58:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/last_update' 2026-02-22T22:58:58.591 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:58:56' 2026-02-22T22:58:58.591 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/state' 2026-02-22T22:58:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:58:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:58:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:58:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:58:58.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:58.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:58:58.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_image cluster2 mirror test_RENAMED test 2026-02-22T22:58:58.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:58.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:58.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_RENAMED 2026-02-22T22:58:58.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_name=test 2026-02-22T22:58:58.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rename mirror/test_RENAMED mirror/test 2026-02-22T22:58:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:58:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T22:58:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:58:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T22:58:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 9 2026-02-22T22:58:59.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:58:59.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:58:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:58:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:58:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:58:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:58:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:58:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:58:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:59:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:59:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:59:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:59:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:59:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:00.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:00.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:00.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:00.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:59:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:59:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:59:01.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:01.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:59:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:01.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:59:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:59:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:59:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:59:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:59:01.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:59:01.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:59:01.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:59:01.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:59:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:59:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:59:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:59:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:59:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:59:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: test trash move restore' 2026-02-22T22:59:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T22:59:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T22:59:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 22:59:01 TEST: test trash move restore 2026-02-22T22:59:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 22:59:01 TEST: test trash move restore 2026-02-22T22:59:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster2 mirror test 2026-02-22T22:59:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:59:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T22:59:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster2 info mirror/test 2026-02-22T22:59:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:59:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id=10b8ad6ebe7d 2026-02-22T22:59:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ trash_move cluster2 mirror test 2026-02-22T22:59:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 trash move mirror/test 2026-02-22T22:59:01.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-22T22:59:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T22:59:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T22:59:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:59:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:59:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:59:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T22:59:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:59:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:59:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:59:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id= 2026-02-22T22:59:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:59:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T22:59:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-02-22T22:59:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T22:59:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T22:59:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T22:59:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T22:59:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:59:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T22:59:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:59:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:59:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:59:02.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T22:59:02.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:59:02.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T22:59:02.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ trash_restore cluster2 mirror test 10b8ad6ebe7d snapshot 2026-02-22T22:59:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=10b8ad6ebe7d 2026-02-22T22:59:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T22:59:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 trash restore mirror/10b8ad6ebe7d 2026-02-22T22:59:02.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:59:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T22:59:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-22T22:59:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:02.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T22:59:02.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-22T22:59:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T22:59:02.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test': 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 10b8ad6ebe7d 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.10b8ad6ebe7d 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 22:57:44 2026 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 22:57:44 2026 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 22:57:44 2026 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: e783395d-e21b-4d02-a753-2fc326654560 2026-02-22T22:59:02.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T22:59:02.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:02.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:59:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:03.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:59:03.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:03.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:03.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:03.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:03.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:04.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:04.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:59:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:59:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:59:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:59:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:59:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:59:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:59:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:59:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:59:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:59:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:59:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:59:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:59:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:59:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:59:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:59:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:59:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:59:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:59:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:59:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:59:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T22:59:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T22:59:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:59:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:04.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T22:59:04.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T22:59:04.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:59:04.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:59:04.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:59:04.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:59:04.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T22:59:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:59:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:59:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T22:59:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:59:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:59:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:59:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:59:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T22:59:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:59:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:59:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:59:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T22:59:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:59:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:59:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:05.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:59:05.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:59:05.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:59:05.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:05.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed (with rbd-mirror on one cluster)' 2026-02-22T22:59:05.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T22:59:05.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T22:59:05.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 22:59:05 TEST: check if removed 'images'\''' OMAP are removed '(with' rbd-mirror on one 'cluster)' 2026-02-22T22:59:05.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 22:59:05 TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster) 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:05.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-22T22:59:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-22T22:59:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:59:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-22T22:59:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-22T22:59:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:59:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=status_global 2026-02-22T22:59:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:59:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:59:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:59:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-22T22:59:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:59:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=status_global 2026-02-22T22:59:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T22:59:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:59:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:59:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c status_global 2026-02-22T22:59:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T22:59:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T22:59:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:59:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:59:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-22T22:59:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:59:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=image_ 2026-02-22T22:59:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:59:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:59:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:59:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-22T22:59:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:59:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_ 2026-02-22T22:59:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T22:59:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:59:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:59:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_ 2026-02-22T22:59:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T22:59:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=image_map 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:59:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_map 2026-02-22T22:59:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T22:59:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:59:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T22:59:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_map 2026-02-22T22:59:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 1 = 0 2026-02-22T22:59:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=1 2026-02-22T22:59:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:59:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T22:59:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:59:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:59:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T22:59:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:59:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_map 2026-02-22T22:59:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T22:59:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:59:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T22:59:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_map 2026-02-22T22:59:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 1 = 0 2026-02-22T22:59:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=1 2026-02-22T22:59:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:59:07.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T22:59:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:59:07.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T22:59:09.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T22:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_map 2026-02-22T22:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T22:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T22:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_map 2026-02-22T22:59:09.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T22:59:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T22:59:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:59:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:59:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-22T22:59:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-22T22:59:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=status_global 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=status_global 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T22:59:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:59:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:59:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:59:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c status_global 2026-02-22T22:59:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T22:59:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=image_ 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:59:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_ 2026-02-22T22:59:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T22:59:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:59:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:59:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:59:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_ 2026-02-22T22:59:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T22:59:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T22:59:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:59:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:59:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=image_map 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:59:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_map 2026-02-22T22:59:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-22T22:59:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:59:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T22:59:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:59:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T22:59:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_map 2026-02-22T22:59:09.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:error getting omap key set mirror/rbd_mirror_leader: (2) No such file or directory 2026-02-22T22:59:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T22:59:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T22:59:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T22:59:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:59:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 2026-02-22T22:59:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:59:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T22:59:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T22:59:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test 2026-02-22T22:59:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:59:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T22:59:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-22T22:59:09.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T22:59:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-22T22:59:09.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T22:59:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-22T22:59:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test': 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 114b5d8fbe2c 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.114b5d8fbe2c 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 22:59:09 2026 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 22:59:09 2026 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 22:59:09 2026 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T22:59:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:59:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:59:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:59:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:59:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:59:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:59:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:10.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:10.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:10.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:10.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:10.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:59:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:59:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:59:11.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:59:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T22:59:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:59:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T22:59:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:11.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T22:59:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:11.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:11.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:11.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T22:59:11.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:59:11.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:11.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:11.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T22:59:11.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T22:59:11.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:59:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:59:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:59:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:59:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:59:11.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:59:11.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:59:11.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:59:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T22:59:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T22:59:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:59:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T22:59:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T22:59:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T22:59:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:59:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:59:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:59:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T22:59:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T22:59:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:59:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T22:59:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T22:59:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:59:11.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:59:11.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:59:11.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:59:11.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T22:59:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:59:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:59:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T22:59:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T22:59:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:12.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:12.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:12.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:59:12.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:59:12.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:12.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:59:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:59:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:59:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test 100 2026-02-22T22:59:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T22:59:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T22:59:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:59:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T22:59:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:59:12.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:59:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:59:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 12500.1 bytes/sec: 49 MiB/s 2026-02-22T22:59:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T22:59:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:59:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:59:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:59:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T22:59:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T22:59:12.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 12 2026-02-22T22:59:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T22:59:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:59:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T22:59:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T22:59:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:59:12.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:59:12.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T22:59:12.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T22:59:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:59:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:59:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:59:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T22:59:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:59:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:59:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=12 2026-02-22T22:59:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T22:59:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:59:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T22:59:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T22:59:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:59:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:59:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=11 2026-02-22T22:59:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 12 = 11 2026-02-22T22:59:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:59:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T22:59:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T22:59:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:59:13.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:59:13.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=11 2026-02-22T22:59:13.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 12 = 11 2026-02-22T22:59:13.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:59:13.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T22:59:14.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:14.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:14.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:14.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:14.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:14.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T22:59:14.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:59:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:59:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=12 2026-02-22T22:59:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 12 = 12 2026-02-22T22:59:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:59:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:59:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:59:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:59:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:59:15.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:15.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:15.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:59:15.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801149,"replay_state":"idle"} 2026-02-22T22:59:15.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:15.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:11' 2026-02-22T22:59:15.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:59:15.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:59:15.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:15.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801149,"replay_state":"idle"} 2026-02-22T22:59:15.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:15.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:59:15.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:59:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:11' 2026-02-22T22:59:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:59:11' 2026-02-22T22:59:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:15.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:15.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:15.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:59:15.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:11' 2026-02-22T22:59:15.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:15.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:15.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801149,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:59:11' 2026-02-22T22:59:15.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:15.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:11' 2026-02-22T22:59:15.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:15.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:59:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:59:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:59:15.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:59:15.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:15.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:15.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: failover and failback' 2026-02-22T22:59:15.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T22:59:15.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T22:59:15.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 22:59:15 TEST: failover and failback 2026-02-22T22:59:15.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 22:59:15 TEST: failover and failback 2026-02-22T22:59:15.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirrors cluster2 2026-02-22T22:59:15.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:15.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T22:59:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster2:0 2026-02-22T22:59:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:0 2026-02-22T22:59:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:59:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:59:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster2:0 2026-02-22T22:59:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:59:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:59:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster2 2026-02-22T22:59:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T22:59:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:59:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster2 2026-02-22T22:59:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T22:59:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T22:59:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:59:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:59:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster2:1 2026-02-22T22:59:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:1 2026-02-22T22:59:15.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster2:1 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster2 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster2 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T22:59:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:59:15.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:59:15.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:15.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster2:2 2026-02-22T22:59:15.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:2 2026-02-22T22:59:15.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:59:15.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:59:15.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster2:2 2026-02-22T22:59:15.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster2 2026-02-22T22:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T22:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster2 2026-02-22T22:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T22:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T22:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T22:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:59:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:59:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster2:3 2026-02-22T22:59:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:3 2026-02-22T22:59:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T22:59:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:59:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster2:3 2026-02-22T22:59:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T22:59:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T22:59:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster2 2026-02-22T22:59:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T22:59:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:59:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster2 2026-02-22T22:59:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T22:59:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T22:59:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T22:59:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:59:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:59:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T22:59:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T22:59:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T22:59:16.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:16.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T22:59:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:59:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:59:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:59:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:59:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:17.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:17.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:17.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:17.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:59:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:59:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:59:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T22:59:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T22:59:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:59:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:18 2026-02-22T22:59:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T22:59:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:17' 2026-02-22T22:59:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:59:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T22:59:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T22:59:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:18' 2026-02-22T22:59:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 22:59:18' 2026-02-22T22:59:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:18' 2026-02-22T22:59:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 22:59:18cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 22:59:17' 2026-02-22T22:59:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:18' 2026-02-22T22:59:19.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T22:59:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:19.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T22:59:19.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T22:59:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T22:59:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T22:59:20.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:20.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:17 2026-02-22T22:59:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T22:59:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:18' 2026-02-22T22:59:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:59:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T22:59:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T22:59:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T22:59:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T22:59:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T22:59:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:22.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:22.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T22:59:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:20 2026-02-22T22:59:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T22:59:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:18' 2026-02-22T22:59:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:59:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T22:59:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T22:59:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:22.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:20' 2026-02-22T22:59:22.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 22:59:20' 2026-02-22T22:59:22.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:20' 2026-02-22T22:59:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 22:59:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 22:59:18' 2026-02-22T22:59:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:22.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:20' 2026-02-22T22:59:22.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T22:59:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:22.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T22:59:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T22:59:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T22:59:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T22:59:23.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T22:59:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:59:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:59:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:59:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:59:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:23.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:59:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:59:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:59:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:25.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:59:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:59:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:59:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test 100 2026-02-22T22:59:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T22:59:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T22:59:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T22:59:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T22:59:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:59:25.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:59:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:59:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 9091 bytes/sec: 36 MiB/s 2026-02-22T22:59:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T22:59:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:59:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:59:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:59:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T22:59:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T22:59:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 15 2026-02-22T22:59:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T22:59:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:59:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:26.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T22:59:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T22:59:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:59:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:59:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T22:59:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T22:59:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:59:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:59:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:59:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T22:59:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:59:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:59:26.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=15 2026-02-22T22:59:26.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T22:59:26.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:59:26.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T22:59:26.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:26.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:26.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:26.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:26.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:26.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T22:59:26.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:59:26.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:59:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=15 2026-02-22T22:59:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 15 = 15 2026-02-22T22:59:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T22:59:26.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:59:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:26.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:27.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T22:59:27.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:27.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:27.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:27.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:59:27.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T22:59:27.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:27.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:59:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T22:59:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:25 2026-02-22T22:59:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T22:59:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:59:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801155,"remote_snapshot_timestamp":1771801162,"replay_state":"idle"} 2026-02-22T22:59:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:23 2026-02-22T22:59:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T22:59:27.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 14 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.e21240d4-0c96-4583-9d87-a9141fe55ba3 (peer_uuids:[]) 2026-02-22T22:59:27.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 15 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.6156f3a8-086b-43eb-bd2a-12a3dce74099 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T22:59:27.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:59:27.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T22:59:27.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:27.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T22:59:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:59:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:59:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:25' 2026-02-22T22:59:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 22:59:25' 2026-02-22T22:59:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:59:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:25' 2026-02-22T22:59:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 22:59:25cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801155,"remote_snapshot_timestamp":1771801162,"replay_state":"idle"}2026-02-22 22:59:23' 2026-02-22T22:59:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:25' 2026-02-22T22:59:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:27.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T22:59:27.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:59:27.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T22:59:27.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:59:27.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T22:59:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:59:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:59:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:59:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:59:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:28.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:59:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:28.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:59:28.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:28.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:28.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:59:28.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801155,"remote_snapshot_timestamp":1771801162,"replay_state":"idle"} 2026-02-22T22:59:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:23 2026-02-22T22:59:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T22:59:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:59:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T22:59:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:25' 2026-02-22T22:59:28.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:59:28.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:59:28.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:28.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801155,"remote_snapshot_timestamp":1771801162,"replay_state":"idle"} 2026-02-22T22:59:28.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:28.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:59:28.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:28.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:59:28.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:28.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:28.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:23' 2026-02-22T22:59:28.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:59:23' 2026-02-22T22:59:28.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:28.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:28.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:28.538 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:59:28.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:23' 2026-02-22T22:59:28.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:28.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:28.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:28.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801155,"remote_snapshot_timestamp":1771801162,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:59:23cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 22:59:25' 2026-02-22T22:59:28.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:28.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:28.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:23' 2026-02-22T22:59:28.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:59:28.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:59:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:59:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:59:28.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:28.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:28.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test 2026-02-22T22:59:28.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:28.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:28.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T22:59:28.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T22:59:28.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.export 2026-02-22T22:59:28.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:59:28.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T22:59:28.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T22:59:28.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.export 2026-02-22T22:59:28.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:59:28.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:59:28.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:59:28.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T22:59:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:59:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:59:28.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:59:28.766 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:59:28.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:59:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T22:59:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T22:59:30.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T22:59:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T22:59:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T22:59:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:59:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:59:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T22:59:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:31.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:31.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:31.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:31.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:31.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:31.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:31.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:31.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:31.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:31.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:31.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:31.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:31.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:31.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:31.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:59:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:59:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:59:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T22:59:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T22:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:59:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T22:59:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:30 2026-02-22T22:59:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T22:59:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:30' 2026-02-22T22:59:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:59:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T22:59:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:32.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image demoted 2026-02-22T22:59:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:32.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:32.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:32.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:30' 2026-02-22T22:59:32.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 22:59:30' 2026-02-22T22:59:32.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:32.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:32.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:32.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:30' 2026-02-22T22:59:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:32.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image demoted43224326mirror.0trial0822026-02-22 22:59:30cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 22:59:30' 2026-02-22T22:59:32.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:32.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:32.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:30' 2026-02-22T22:59:32.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T22:59:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T22:59:32.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:32.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:32.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:32.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T22:59:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T22:59:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T22:59:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:30 2026-02-22T22:59:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T22:59:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T22:59:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:30' 2026-02-22T22:59:33.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:59:33.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T22:59:33.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:33.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T22:59:33.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:33.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:33.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:33.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:33.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:33.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:33.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:33.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:33.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:30' 2026-02-22T22:59:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 22:59:30' 2026-02-22T22:59:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:30' 2026-02-22T22:59:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 22:59:30cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image demoted2026-02-22 22:59:30' 2026-02-22T22:59:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:33.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:30' 2026-02-22T22:59:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T22:59:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T22:59:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T22:59:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T22:59:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T22:59:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:34.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T22:59:34.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T22:59:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:34.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:34.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:34.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:59:34.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:59:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T22:59:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T22:59:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:59:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:59:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T22:59:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:34.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:59:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:59:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:59:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:59:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:35.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:35.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:59:35.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:59:35.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:35.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T22:59:35.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:35.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T22:59:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T22:59:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:59:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T22:59:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T22:59:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:59:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T22:59:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:37.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:59:37.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T22:59:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T22:59:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:59:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T22:59:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:59:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:59:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:59:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:59:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:59:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:59:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:59:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:59:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:59:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T22:59:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T22:59:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T22:59:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T22:59:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T22:59:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:40.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:40.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:40.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:59:40.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:59:40.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:40.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T22:59:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:59:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T22:59:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T22:59:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T22:59:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T22:59:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:59:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T22:59:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T22:59:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:59:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:59:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T22:59:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:59:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:59:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:59:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:59:41.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:59:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:59:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:59:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T22:59:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T22:59:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:59:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:38 2026-02-22T22:59:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T22:59:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:40' 2026-02-22T22:59:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:59:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T22:59:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T22:59:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:38' 2026-02-22T22:59:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 22:59:38' 2026-02-22T22:59:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:38' 2026-02-22T22:59:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 22:59:38cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 22:59:40' 2026-02-22T22:59:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:38' 2026-02-22T22:59:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:43.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T22:59:43.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:43.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T22:59:43.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T22:59:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T22:59:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T22:59:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:40 2026-02-22T22:59:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T22:59:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T22:59:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T22:59:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:43' 2026-02-22T22:59:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:59:44.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T22:59:44.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:44.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T22:59:44.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:44.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:44.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:44.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:44.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:40' 2026-02-22T22:59:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 22:59:40' 2026-02-22T22:59:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:40' 2026-02-22T22:59:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:44.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 22:59:40cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 22:59:43' 2026-02-22T22:59:44.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:44.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:44.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:40' 2026-02-22T22:59:44.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T22:59:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T22:59:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T22:59:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T22:59:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T22:59:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T22:59:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T22:59:45.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T22:59:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:46.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:59:46.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:46.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:46.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:46.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:59:46.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:46.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:59:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:59:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:59:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:46.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:46.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:46.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:46.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:59:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:59:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T22:59:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T22:59:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:59:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T22:59:49.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:59:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:59:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:59:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:49.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:49.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:49.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:49.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:49.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:59:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:59:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:59:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T22:59:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T22:59:52.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 20 2026-02-22T22:59:52.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T22:59:52.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T22:59:52.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T22:59:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T22:59:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:59:52.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:59:52.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T22:59:52.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T22:59:52.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:59:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:59:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T22:59:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T22:59:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:59:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:59:53.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=20 2026-02-22T22:59:53.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T22:59:53.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:59:53.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T22:59:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T22:59:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T22:59:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:59:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:59:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=20 2026-02-22T22:59:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 20 = 20 2026-02-22T22:59:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T22:59:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:59:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801189,"remote_snapshot_timestamp":1771801189,"replay_state":"idle"} 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:53 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:50' 2026-02-22T22:59:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:59:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T22:59:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801189,"remote_snapshot_timestamp":1771801189,"replay_state":"idle"} 2026-02-22T22:59:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:59:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:59:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:53' 2026-02-22T22:59:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:59:53' 2026-02-22T22:59:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T22:59:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:53' 2026-02-22T22:59:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:54.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:54.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:54.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801189,"remote_snapshot_timestamp":1771801189,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 22:59:53cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 22:59:50' 2026-02-22T22:59:54.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:54.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:53' 2026-02-22T22:59:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T22:59:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T22:59:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T22:59:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T22:59:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T22:59:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:59:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T22:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:55.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:55.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:55.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T22:59:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T22:59:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T22:59:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T22:59:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T22:59:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T22:59:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:50 2026-02-22T22:59:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T22:59:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T22:59:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T22:59:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801189,"remote_snapshot_timestamp":1771801189,"replay_state":"idle"} 2026-02-22T22:59:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:53 2026-02-22T22:59:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T22:59:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 20 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.17d92d97-b201-4c7b-945e-1655384f9cf8 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T22:59:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:59:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T22:59:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T22:59:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T22:59:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T22:59:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:59:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T22:59:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:59:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:59:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T22:59:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T22:59:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:59:55.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:50' 2026-02-22T22:59:55.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 22:59:50' 2026-02-22T22:59:55.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:55.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:55.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:55.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T22:59:55.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:50' 2026-02-22T22:59:55.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T22:59:55.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T22:59:55.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:59:55.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 22:59:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801189,"remote_snapshot_timestamp":1771801189,"replay_state":"idle"}2026-02-22 22:59:53' 2026-02-22T22:59:55.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T22:59:55.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:59:55.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:50' 2026-02-22T22:59:55.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:59:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T22:59:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T22:59:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T22:59:55.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T22:59:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test 2026-02-22T22:59:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:55.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:55.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T22:59:55.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T22:59:55.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.export 2026-02-22T22:59:55.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:59:55.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T22:59:55.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T22:59:55.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.export 2026-02-22T22:59:55.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:59:55.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:59:55.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:59:55.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T22:59:55.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:59:55.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:59:55.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:59:55.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:59:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T22:59:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T22:59:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T22:59:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T22:59:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T22:59:55.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T22:59:55.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:55.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:55.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:55.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T22:59:55.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:55.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:55.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:55.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T22:59:55.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T22:59:55.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:55.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T22:59:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T22:59:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:56.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:56.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:56.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:56.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:56.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:59:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:59:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T22:59:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T22:59:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T22:59:57.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:57.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T22:59:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T22:59:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T22:59:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T22:59:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T22:59:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:59:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T22:59:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T22:59:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T22:59:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T22:59:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T22:59:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T22:59:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:59:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T22:59:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:59:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:59:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T22:59:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T22:59:59.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:59:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T22:59:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T22:59:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T22:59:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:59:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:59:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:59:59.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T22:59:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:59:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:59:59.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T22:59:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T22:59:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T22:59:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T22:59:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T22:59:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T22:59:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T22:59:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T22:59:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T22:59:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T22:59:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:59:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:00:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:00:00.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:00.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:00.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:00.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:58 2026-02-22T23:00:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:00:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:00:00.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:55' 2026-02-22T23:00:00.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:00:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:00:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:00:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:58' 2026-02-22T23:00:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 22:59:58' 2026-02-22T23:00:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 22:59:58' 2026-02-22T23:00:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 22:59:58cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 22:59:55' 2026-02-22T23:00:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:00.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 22:59:58' 2026-02-22T23:00:00.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:00.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:00:00.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:00.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:00:00.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:00:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:00:01.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:00 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 22:59:58' 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:00:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:00:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:01.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:00' 2026-02-22T23:00:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:00:00' 2026-02-22T23:00:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:00' 2026-02-22T23:00:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:01.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:00:00cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 22:59:58' 2026-02-22T23:00:01.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:01.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:00' 2026-02-22T23:00:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:00:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:00:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:01.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:01.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:01.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:00:01.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:01.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:01.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:01.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:00:01.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:00:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:00:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:00:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:00:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:00:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:00:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:00:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:00:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:00:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:00:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:00:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:00:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:00:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:00:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:00:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:00:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:00:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:00:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:00:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:00:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:00:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:00:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:00:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:00:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:00:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:00:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:00:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:00:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:00:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:00:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:00:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:00:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:00:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:00:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:00:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:00:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:00:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:05.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:00:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:00:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:00:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:00:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:00:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:00:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:00:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:00:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:00:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:00:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:00:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:00:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:00:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:00:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:00:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:00:09.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:00:09.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:09.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:00:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:00:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:00:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster1 mirror test 100 2026-02-22T23:00:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:00:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:00:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:00:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:00:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:00:09.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:00:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:00:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 7692.39 bytes/sec: 30 MiB/s 2026-02-22T23:00:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:00:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:00:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:00:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:00:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:00:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:00:12.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 23 2026-02-22T23:00:12.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:00:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:00:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:12.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:00:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:00:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:00:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:00:12.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:00:12.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:00:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:00:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=23 2026-02-22T23:00:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:00:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:00:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:00:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:00:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:00:13.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:00:13.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=23 2026-02-22T23:00:13.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 23 = 23 2026-02-22T23:00:13.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:13.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:13.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:00:14.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:00:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:00:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:13 2026-02-22T23:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801201,"replay_state":"idle"} 2026-02-22T23:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:05 2026-02-22T23:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 23 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.b40246b1-a99d-4212-815a-3a376c801a57 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:00:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:00:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:00:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:00:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:00:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:00:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:14.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:13' 2026-02-22T23:00:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:00:13' 2026-02-22T23:00:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:13' 2026-02-22T23:00:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:14.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:00:13cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801201,"replay_state":"idle"}2026-02-22 23:00:05' 2026-02-22T23:00:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:13' 2026-02-22T23:00:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:00:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:00:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:00:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:00:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror test 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:00:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:00:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:00:15.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:15.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:15.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:00:15.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801201,"replay_state":"idle"} 2026-02-22T23:00:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:05 2026-02-22T23:00:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:00:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:00:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:00:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:13' 2026-02-22T23:00:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:00:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:00:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801201,"replay_state":"idle"} 2026-02-22T23:00:15.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:15.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:00:15.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:15.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:00:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:05' 2026-02-22T23:00:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:00:05' 2026-02-22T23:00:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:00:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:05' 2026-02-22T23:00:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:15.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801201,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:00:05cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:00:13' 2026-02-22T23:00:15.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:05' 2026-02-22T23:00:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:15.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:00:15.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:00:15.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:00:15.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:00:15.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:15.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:15.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test 2026-02-22T23:00:15.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:15.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:15.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:00:15.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T23:00:15.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.export 2026-02-22T23:00:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T23:00:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T23:00:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.export 2026-02-22T23:00:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:00:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:00:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T23:00:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:00:15.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:00:15.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:00:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:00:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:00:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:00:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:00:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:00:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:00:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:00:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:00:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:00:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:16.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:00:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:00:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:00:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:00:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:00:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:00:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:00:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:16.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:00:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:00:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:00:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:00:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:00:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:00:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:00:18.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:18.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:13 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:16' 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:00:20.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:00:20.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:20.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:00:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:18 2026-02-22T23:00:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:00:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:00:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:16' 2026-02-22T23:00:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:00:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:00:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:20.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:00:20.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:20.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:20.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:20.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:20.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:18' 2026-02-22T23:00:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:00:18' 2026-02-22T23:00:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:18' 2026-02-22T23:00:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:00:18cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:00:16' 2026-02-22T23:00:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:20.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:20.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:18' 2026-02-22T23:00:20.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:00:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:00:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:00:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:20.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:00:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:00:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:20 2026-02-22T23:00:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:00:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:21.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:18' 2026-02-22T23:00:21.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:00:21.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:00:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:00:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:21.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:21.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:21.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:20' 2026-02-22T23:00:21.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:00:20' 2026-02-22T23:00:21.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:21.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:21.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:21.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:21.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:20' 2026-02-22T23:00:21.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:21.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:21.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:21.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:00:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:00:18' 2026-02-22T23:00:21.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:21.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:20' 2026-02-22T23:00:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:21.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:00:21.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:21.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:00:21.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:00:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:00:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:00:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:00:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:00:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:00:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:00:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:00:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:00:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:00:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:00:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:00:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:00:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:00:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:00:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:00:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:00:23.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:24.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:24.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:24.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:00:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:00:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test 100 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:00:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:00:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:00:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:00:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 7692.39 bytes/sec: 30 MiB/s 2026-02-22T23:00:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:00:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:00:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:00:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:00:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:00:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:24.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:00:27.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 26 2026-02-22T23:00:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:00:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:00:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:00:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:00:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:00:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:00:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:00:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:00:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:00:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=26 2026-02-22T23:00:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:00:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:00:28.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:00:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:00:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:00:28.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:00:28.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=26 2026-02-22T23:00:28.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 26 = 26 2026-02-22T23:00:28.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:29.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:00:29.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:29.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:29.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:29.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:00:29.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:29.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:29.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:29.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:29.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:00:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:00:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801221,"replay_state":"idle"} 2026-02-22T23:00:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:23 2026-02-22T23:00:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:00:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:00:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:00:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:25' 2026-02-22T23:00:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:00:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:00:29.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:29.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801221,"replay_state":"idle"} 2026-02-22T23:00:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:00:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:29.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:00:29.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:29.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:23' 2026-02-22T23:00:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:00:23' 2026-02-22T23:00:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:00:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:23' 2026-02-22T23:00:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801221,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:00:23cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:00:25' 2026-02-22T23:00:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:29.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:23' 2026-02-22T23:00:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:00:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:00:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:00:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:00:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:00:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:00:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:00:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:25 2026-02-22T23:00:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:00:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:00:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801221,"replay_state":"idle"} 2026-02-22T23:00:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:23 2026-02-22T23:00:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:00:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 26 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.3c9beabd-bed6-4896-bcd7-37416524c089 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:00:30.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:00:30.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:00:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:30.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:00:30.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:00:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:00:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:25' 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:00:25' 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:25' 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:00:25cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801221,"replay_state":"idle"}2026-02-22 23:00:23' 2026-02-22T23:00:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:25' 2026-02-22T23:00:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:30.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:00:30.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:00:30.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:00:30.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:00:30.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test 2026-02-22T23:00:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:00:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T23:00:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.export 2026-02-22T23:00:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T23:00:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T23:00:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.export 2026-02-22T23:00:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:30.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:00:30.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:00:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T23:00:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:00:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:00:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:00:30.591 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:00:30.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:00:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: failover / failback loop' 2026-02-22T23:00:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:00:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:00:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:00:30 TEST: failover / failback loop 2026-02-22T23:00:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:00:30 TEST: failover / failback loop 2026-02-22T23:00:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 1 20 2026-02-22T23:00:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:00:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:00:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:00:30.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:00:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:00:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:00:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:00:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:00:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:31.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:00:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:00:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:00:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:00:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:00:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:00:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:00:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:00:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:00:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:00:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:00:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:00:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:00:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:00:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:00:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:00:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:00:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:00:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:00:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:00:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:00:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:00:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:00:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:00:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:00:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:00:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:00:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:00:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:00:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:00:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:00:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:00:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:00:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:00:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:00:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:00:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:00:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:00:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:00:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:00:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:00:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:33 2026-02-22T23:00:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:00:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:00:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:00:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:30' 2026-02-22T23:00:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:00:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:00:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:00:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:33' 2026-02-22T23:00:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:00:33' 2026-02-22T23:00:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:33' 2026-02-22T23:00:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:00:33cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:00:30' 2026-02-22T23:00:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:33' 2026-02-22T23:00:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:00:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:00:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:35 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:33' 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:00:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:00:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:00:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:35' 2026-02-22T23:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:00:35' 2026-02-22T23:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:35' 2026-02-22T23:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:00:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:00:33' 2026-02-22T23:00:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:35' 2026-02-22T23:00:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:36.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:00:36.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:36.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:00:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:36.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:36.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:00:36.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:36.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:36.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:36.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:00:36.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:00:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:00:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:00:37.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:00:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:00:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:00:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:37.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:00:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:00:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:00:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:00:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:00:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:00:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:38.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:38.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:00:38.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:38.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:38.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:00:38.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:00:38.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:00:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:00:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:00:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:38.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:38.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:38.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:38.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:00:38.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:00:38.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:00:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:00:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:00:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:00:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:00:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:00:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:41.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:41.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:41.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:41.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:00:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:00:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:41.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:00:41.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:00:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:41.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:41.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:41.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:00:44.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 29 2026-02-22T23:00:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:00:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:00:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:00:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:00:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:00:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:00:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:00:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:00:44.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:00:44.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=29 2026-02-22T23:00:44.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:00:44.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:00:44.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:00:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:00:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:00:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:00:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=29 2026-02-22T23:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 29 = 29 2026-02-22T23:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:45.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:45.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:46.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:46.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:00:46.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:00:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:00:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:43 2026-02-22T23:00:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:00:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:00:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801241,"remote_snapshot_timestamp":1771801241,"replay_state":"idle"} 2026-02-22T23:00:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:45 2026-02-22T23:00:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:00:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 29 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.2230a8a7-0c98-43d1-9d10-58b62aea4b40 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:00:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:00:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:00:46.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:46.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:00:46.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:46.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:00:46.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:46.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:00:46.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:46.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:46.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:46.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:46.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:43' 2026-02-22T23:00:46.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:00:43' 2026-02-22T23:00:46.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:46.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:46.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:46.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:46.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:43' 2026-02-22T23:00:46.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:46.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:46.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:00:43cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801241,"remote_snapshot_timestamp":1771801241,"replay_state":"idle"}2026-02-22 23:00:45' 2026-02-22T23:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:46.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:43' 2026-02-22T23:00:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:00:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:00:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:00:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:00:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:00:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:46.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:00:46.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:46.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:46.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:46.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:00:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:00:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:00:47.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:47.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:47.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801241,"remote_snapshot_timestamp":1771801241,"replay_state":"idle"} 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:45 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:43' 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:00:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:47.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801241,"remote_snapshot_timestamp":1771801241,"replay_state":"idle"} 2026-02-22T23:00:47.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:47.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:00:47.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:47.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:00:47.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:47.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:47.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:47.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:45' 2026-02-22T23:00:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:00:45' 2026-02-22T23:00:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:00:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:45' 2026-02-22T23:00:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:47.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:47.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801241,"remote_snapshot_timestamp":1771801241,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:00:45cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:00:43' 2026-02-22T23:00:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:47.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:45' 2026-02-22T23:00:47.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:47.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:00:47.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:00:47.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:00:47.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:00:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:00:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:00:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:00:47.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:00:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:00:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:00:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:00:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:48.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:00:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:00:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:00:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:00:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:00:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:00:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:00:48.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:48.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:48.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:00:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:00:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:49.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:00:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:00:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:00:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:00:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:00:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:00:50.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:48 2026-02-22T23:00:50.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:00:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:00:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:48' 2026-02-22T23:00:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:00:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:00:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:00:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:50.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:50.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:50.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:50.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:48' 2026-02-22T23:00:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:00:48' 2026-02-22T23:00:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:48' 2026-02-22T23:00:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:00:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:00:48' 2026-02-22T23:00:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:50.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:48' 2026-02-22T23:00:50.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:00:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:00:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:00:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:00:51.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:51.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:51.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:51.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:50 2026-02-22T23:00:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:00:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:00:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:00:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:48' 2026-02-22T23:00:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:00:51.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:00:51.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:51.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:00:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:50' 2026-02-22T23:00:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:00:50' 2026-02-22T23:00:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:00:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:50' 2026-02-22T23:00:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:51.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:00:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:00:48' 2026-02-22T23:00:51.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:51.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:51.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:50' 2026-02-22T23:00:51.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:00:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:00:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:00:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:00:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:00:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:00:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:00:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:00:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:52.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:00:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:00:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:00:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:00:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:00:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:00:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:00:53.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:53.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:00:53.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:00:53.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:00:53.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:53.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:00:53.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:00:53.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:00:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:00:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:00:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:00:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:00:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:00:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:00:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:00:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:00:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:00:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:00:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:00:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:00:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:00:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:00:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:00:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:00:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:00:54.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:54.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:54.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:54.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:00:57.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 32 2026-02-22T23:00:57.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:00:57.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:00:57.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:57.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:57.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:57.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:00:57.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:00:57.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:57.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:57.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:00:57.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:00:57.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:00:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:00:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:00:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:00:58.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:00:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=32 2026-02-22T23:00:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:00:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:00:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:00:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:00:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:00:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:00:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:00:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=32 2026-02-22T23:00:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 32 = 32 2026-02-22T23:00:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:00:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:00:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:00:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:00:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:00:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:00:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:00:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:00:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:00:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:55 2026-02-22T23:00:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:00:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:00:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:00:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801251,"replay_state":"idle"} 2026-02-22T23:00:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:53 2026-02-22T23:00:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:00:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 32 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.747afbe2-fcc4-432c-8970-72f4ccfd7f01 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:00:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:00:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:00:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:00:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:00:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:00:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:00:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:00:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:00:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:00:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:00:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:00:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:00:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:55' 2026-02-22T23:00:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:00:55' 2026-02-22T23:00:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:00:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:00:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:55' 2026-02-22T23:00:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:00:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:00:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:00:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:00:55cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801251,"replay_state":"idle"}2026-02-22 23:00:53' 2026-02-22T23:00:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:00:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:00:59.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:55' 2026-02-22T23:00:59.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:00:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:00:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:00:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:00:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:00:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:00:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:00:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:00:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:00:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:00:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:00:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:00:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:00:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:00:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:01:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:00.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:00.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801251,"replay_state":"idle"} 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:53 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:00:55' 2026-02-22T23:01:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:01:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:01:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801251,"replay_state":"idle"} 2026-02-22T23:01:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:01:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:01:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:53' 2026-02-22T23:01:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:00:53' 2026-02-22T23:01:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:00:53' 2026-02-22T23:01:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801251,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:00:53cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:00:55' 2026-02-22T23:01:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:00:53' 2026-02-22T23:01:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:01:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:01:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:01:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:01:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:01:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:01:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:01:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:01:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:01:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:01:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:01:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:01:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:01:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:01:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:01.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:01:01.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:01.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:01:01.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:01:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:01:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:02.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:01:02.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:02.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:02.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:01:02.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:01:02.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:02.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:01:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:01:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:01:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:01:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:01:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:01:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:01:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:01:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:01:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:01:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:01:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:01:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:01:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:01:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:01:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:01:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:01:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:01:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:01:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:01:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:04.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:01:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:01:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:01:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:01:04.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:04.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:04.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:04.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:01:04.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:01:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:05.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:05.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:05.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:03 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:00' 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:01:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:01:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:01:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:03' 2026-02-22T23:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:01:03' 2026-02-22T23:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:03' 2026-02-22T23:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:01:03cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:01:00' 2026-02-22T23:01:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:03' 2026-02-22T23:01:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:01:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:01:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:01:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:01:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:01:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:05 2026-02-22T23:01:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:01:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:03' 2026-02-22T23:01:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:01:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:01:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:01:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:05' 2026-02-22T23:01:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:01:05' 2026-02-22T23:01:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:05' 2026-02-22T23:01:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:06.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:01:05cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:01:03' 2026-02-22T23:01:06.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:06.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:05' 2026-02-22T23:01:06.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:01:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:01:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:01:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:01:06.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:01:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:01:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:01:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:08.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:01:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:01:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:01:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:01:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:01:08.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:08.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:08.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:01:08.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:08.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:08.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:08.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:08.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:01:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:01:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:01:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:01:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:01:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:01:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:01:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:01:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:01:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:01:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:01:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:11.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:01:11.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:01:11.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:01:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:01:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:01:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:01:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:01:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:01:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:01:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:01:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:01:13.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 35 2026-02-22T23:01:13.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:01:13.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:01:13.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:13.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:13.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:13.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:01:13.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:01:13.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:01:13.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:13.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:01:13.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:01:13.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:01:13.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:01:13.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:01:13.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:01:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=35 2026-02-22T23:01:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:01:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:01:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:01:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:01:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:01:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:01:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=35 2026-02-22T23:01:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 35 = 35 2026-02-22T23:01:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:01:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:14.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:14.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:14.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:14.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:14.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:01:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:01:15.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:13 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801266,"replay_state":"idle"} 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:10 2026-02-22T23:01:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:01:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 34 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.91c60073-547d-42f2-bf59-eeae74b39cc1 (peer_uuids:[]) 2026-02-22T23:01:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 35 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.c59364fd-1d46-42a0-b0d1-081b349d6c61 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:01:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:01:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:01:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:01:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:01:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:01:15.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:15.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:13' 2026-02-22T23:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:01:13' 2026-02-22T23:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:13' 2026-02-22T23:01:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:01:13cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801266,"replay_state":"idle"}2026-02-22 23:01:10' 2026-02-22T23:01:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:13' 2026-02-22T23:01:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:01:15.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:01:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:16.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:01:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:16.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:01:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:01:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801271,"remote_snapshot_timestamp":1771801271,"replay_state":"idle"} 2026-02-22T23:01:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:15 2026-02-22T23:01:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:01:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:13' 2026-02-22T23:01:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:01:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:01:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:16.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801271,"remote_snapshot_timestamp":1771801271,"replay_state":"idle"} 2026-02-22T23:01:16.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:16.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:01:16.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:16.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:01:16.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:16.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:16.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:16.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:16.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:15' 2026-02-22T23:01:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:01:15' 2026-02-22T23:01:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:15' 2026-02-22T23:01:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801271,"remote_snapshot_timestamp":1771801271,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:01:15cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:01:13' 2026-02-22T23:01:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:16.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:16.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:15' 2026-02-22T23:01:16.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:16.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:01:16.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:01:16.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:01:16.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:01:16.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:01:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:01:17.933 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:01:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:01:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:01:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:01:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:01:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:01:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:01:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:01:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:01:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:01:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:01:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:01:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:01:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:01:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:18.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:18.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:01:18.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:18.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:18.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:18.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:19.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:19.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:01:19.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:01:19.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:19.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:01:19.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:19.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:19.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:01:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:01:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:18 2026-02-22T23:01:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:01:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:01:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:17' 2026-02-22T23:01:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:01:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:01:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:01:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:20.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:20.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:20.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:18' 2026-02-22T23:01:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:01:18' 2026-02-22T23:01:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:20.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:20.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:20.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:18' 2026-02-22T23:01:20.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:20.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:20.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:20.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:01:18cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:01:17' 2026-02-22T23:01:20.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:20.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:18' 2026-02-22T23:01:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:01:20.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:20.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:01:20.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:01:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:20.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:01:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:01:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:20 2026-02-22T23:01:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:01:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:18' 2026-02-22T23:01:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:01:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:01:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:01:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:21.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:21.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:21.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:21.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:20' 2026-02-22T23:01:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:01:20' 2026-02-22T23:01:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:20' 2026-02-22T23:01:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:01:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:01:18' 2026-02-22T23:01:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:20' 2026-02-22T23:01:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:01:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:01:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:01:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:01:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:01:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:01:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:01:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:01:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:01:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:01:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:22.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:01:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:01:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:01:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:01:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:01:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:01:23.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:24.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:24.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:24.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:01:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:01:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:01:27.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 38 2026-02-22T23:01:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:01:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:01:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:01:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:01:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:01:27.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:27.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:01:27.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:01:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:01:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:01:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:01:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:01:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=38 2026-02-22T23:01:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:01:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:01:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:01:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:01:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:01:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:01:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=38 2026-02-22T23:01:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 38 = 38 2026-02-22T23:01:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:01:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:28.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:28.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:28.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:28.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:28.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:28.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:28.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:29.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:01:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:01:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:25 2026-02-22T23:01:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:01:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:01:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801281,"replay_state":"idle"} 2026-02-22T23:01:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:23 2026-02-22T23:01:29.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:01:29.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 38 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.c2412795-e7d4-44b5-89e1-7f3b1810bb8d (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:01:29.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:01:29.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:01:29.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:29.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:01:29.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:29.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:01:29.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:29.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:01:29.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:25' 2026-02-22T23:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:01:25' 2026-02-22T23:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:25' 2026-02-22T23:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:01:25cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801281,"replay_state":"idle"}2026-02-22 23:01:23' 2026-02-22T23:01:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:25' 2026-02-22T23:01:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:01:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:01:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:01:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:29.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:01:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:01:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:01:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801281,"replay_state":"idle"} 2026-02-22T23:01:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:23 2026-02-22T23:01:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:01:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:25' 2026-02-22T23:01:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:01:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:01:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801281,"replay_state":"idle"} 2026-02-22T23:01:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:01:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:01:30.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:30.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:30.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:30.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:30.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:23' 2026-02-22T23:01:30.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:01:23' 2026-02-22T23:01:30.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:30.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:30.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:30.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:30.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:23' 2026-02-22T23:01:30.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:30.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:30.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:30.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801281,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:01:23cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:01:25' 2026-02-22T23:01:30.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:30.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:23' 2026-02-22T23:01:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:01:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:01:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:01:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:01:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:01:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:01:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:01:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:01:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:01:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:01:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:01:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:01:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:01:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:01:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:01:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:01:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:01:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:01:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:01:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:01:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:01:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:01:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:01:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:01:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:01:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:01:32.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:01:32.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:32.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:01:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:01:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:01:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:01:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:01:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:01:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:01:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:01:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:01:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:01:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:34.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:01:34.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:34.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:01:34.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:34.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:34.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:34.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:34.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:34.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:34.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:34.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:35.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:01:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:35.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:01:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:33 2026-02-22T23:01:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:01:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:30' 2026-02-22T23:01:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:01:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:01:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:01:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:33' 2026-02-22T23:01:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:01:33' 2026-02-22T23:01:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:33' 2026-02-22T23:01:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:35.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:01:33cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:01:30' 2026-02-22T23:01:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:35.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:33' 2026-02-22T23:01:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:01:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:01:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:01:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:01:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:01:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:35 2026-02-22T23:01:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:01:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:33' 2026-02-22T23:01:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:01:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:01:36.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:01:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:35' 2026-02-22T23:01:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:01:35' 2026-02-22T23:01:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:35' 2026-02-22T23:01:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:36.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:01:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:01:33' 2026-02-22T23:01:36.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:36.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:35' 2026-02-22T23:01:36.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:01:36.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:36.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:01:36.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:36.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:01:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:01:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:01:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:01:37.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:01:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:01:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:01:37.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:01:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:01:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:01:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:01:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:01:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:01:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:01:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:01:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:01:38.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:01:38.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:01:38.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:01:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:01:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:01:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:01:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:01:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:01:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:01:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:01:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:01:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:01:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:01:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:01:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:41.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:01:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:01:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:01:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:01:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:01:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:01:44.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 41 2026-02-22T23:01:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:01:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:01:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:01:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:01:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:01:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:01:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:01:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:01:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:01:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:01:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:01:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=41 2026-02-22T23:01:45.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:01:45.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:01:45.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:01:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:01:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:01:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:01:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=41 2026-02-22T23:01:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 41 = 41 2026-02-22T23:01:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:01:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:46.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:01:46.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:46.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:46.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:01:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:43 2026-02-22T23:01:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:01:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:01:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801301,"remote_snapshot_timestamp":1771801301,"replay_state":"idle"} 2026-02-22T23:01:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:45 2026-02-22T23:01:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:01:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 41 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.220cf2db-3586-4a0c-9b1e-fbe8bb01bc27 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:01:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:01:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:01:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:01:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:01:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:01:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:46.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:46.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:46.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:43' 2026-02-22T23:01:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:01:43' 2026-02-22T23:01:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:43' 2026-02-22T23:01:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:46.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:01:43cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801301,"remote_snapshot_timestamp":1771801301,"replay_state":"idle"}2026-02-22 23:01:45' 2026-02-22T23:01:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:46.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:43' 2026-02-22T23:01:46.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:01:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:01:46.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:01:46.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:01:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:01:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:47.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:01:47.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:47.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:47.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:47.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:47.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:47.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:47.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:47.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:47.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:01:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:01:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801301,"remote_snapshot_timestamp":1771801301,"replay_state":"idle"} 2026-02-22T23:01:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:45 2026-02-22T23:01:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:01:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:43' 2026-02-22T23:01:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:01:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:01:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:47.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801301,"remote_snapshot_timestamp":1771801301,"replay_state":"idle"} 2026-02-22T23:01:47.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:47.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:01:47.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:47.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:01:47.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:47.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:45' 2026-02-22T23:01:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:01:45' 2026-02-22T23:01:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:47.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:45' 2026-02-22T23:01:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:47.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801301,"remote_snapshot_timestamp":1771801301,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:01:45cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:01:43' 2026-02-22T23:01:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:47.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:45' 2026-02-22T23:01:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:01:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:01:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:01:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:01:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:01:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:01:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:01:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:01:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:01:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:01:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:01:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:48.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:01:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:01:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:01:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:01:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:01:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:01:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:01:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:01:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:01:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:01:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:01:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:01:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:01:48.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:48.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:48.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:01:48.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:01:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:01:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:01:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:01:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:49.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:01:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:50.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:50.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:50.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:01:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:48 2026-02-22T23:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:50.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:01:50.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:48' 2026-02-22T23:01:50.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:01:50.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:01:50.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:50.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:01:50.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:48' 2026-02-22T23:01:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:01:48' 2026-02-22T23:01:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:48' 2026-02-22T23:01:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:01:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:01:48' 2026-02-22T23:01:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:50.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:48' 2026-02-22T23:01:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:01:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:01:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:01:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:01:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:50 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:48' 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:01:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:01:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:51.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:51.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:50' 2026-02-22T23:01:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:01:50' 2026-02-22T23:01:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:01:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:50' 2026-02-22T23:01:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:01:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:01:48' 2026-02-22T23:01:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:50' 2026-02-22T23:01:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:01:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:01:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:01:51.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:01:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:01:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:01:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:01:52.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:01:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:01:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:52.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:01:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:01:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:01:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:01:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:01:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:01:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:01:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:01:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:01:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:01:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:01:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:01:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:01:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:53.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:01:53.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:01:53.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:01:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:01:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:01:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:01:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:01:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:01:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:01:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:01:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:01:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:01:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:01:54.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:01:54.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:01:54.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:01:54.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:01:54.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:01:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:01:56.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 44 2026-02-22T23:01:56.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:01:56.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:01:56.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:56.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:56.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:56.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:01:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:01:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:01:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:01:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:01:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:01:57.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:57.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:57.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:57.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:57.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:57.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:01:57.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:01:57.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:01:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:01:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=44 2026-02-22T23:01:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:01:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:01:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:01:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:01:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:01:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:01:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:01:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=44 2026-02-22T23:01:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 44 = 44 2026-02-22T23:01:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:01:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:57.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:58.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:01:58.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:01:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:55 2026-02-22T23:01:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:01:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:01:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801311,"replay_state":"idle"} 2026-02-22T23:01:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:53 2026-02-22T23:01:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:01:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 43 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.1998c98f-a9c3-45a6-bc85-f8df109b45e4 (peer_uuids:[]) 2026-02-22T23:01:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 44 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.7311083a-4aa1-493a-b0ad-b983a7b2b540 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:01:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:01:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:01:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:01:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:01:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:01:58.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:58.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:58.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:58.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:55' 2026-02-22T23:01:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:01:55' 2026-02-22T23:01:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:01:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:55' 2026-02-22T23:01:58.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:58.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:58.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:01:55cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801311,"replay_state":"idle"}2026-02-22 23:01:53' 2026-02-22T23:01:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:58.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:55' 2026-02-22T23:01:58.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:58.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:01:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:01:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:01:58.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:01:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:01:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:01:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:01:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:01:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:01:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:01:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801311,"replay_state":"idle"} 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:53 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:01:55' 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:01:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:01:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:01:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801311,"replay_state":"idle"} 2026-02-22T23:01:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:01:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:01:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:01:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:01:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:01:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:01:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:01:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:01:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:53' 2026-02-22T23:01:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:01:53' 2026-02-22T23:01:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:01:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:01:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:01:53' 2026-02-22T23:01:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:01:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:01:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:01:59.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801311,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:01:53cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:01:55' 2026-02-22T23:01:59.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:01:59.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:01:59.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:01:53' 2026-02-22T23:01:59.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:01:59.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:01:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:01:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:01:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:01:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:01:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:01:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:01:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:01:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:01:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:01:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:02:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:02:01.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:02:01.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:02:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:02:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:02:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:02:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:02:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:02:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:02:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:02:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:02:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:02:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:02:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:02:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:02:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:02:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:02:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:02:02.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:02:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:02:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:02:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:02:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:02:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:02:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:02:02.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:02:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:02:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:02:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:02:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:02:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:02:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:02:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:02:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:02:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:02:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:02:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:02:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:02:02.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:02:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:02:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:02:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:02:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:02:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:02:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:02:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:02:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:02:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:02:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:02:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:02:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:02.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:02:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:02:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:02:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:02 2026-02-22T23:02:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:02:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:00' 2026-02-22T23:02:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:02:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:02:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image demoted 2026-02-22T23:02:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:03.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:02' 2026-02-22T23:02:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:02:02' 2026-02-22T23:02:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:02' 2026-02-22T23:02:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image demoted43224326mirror.0trial0822026-02-22 23:02:02cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:02:00' 2026-02-22T23:02:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:02' 2026-02-22T23:02:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:03.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:02:03.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:03.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:02:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:02:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:02:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:02:04.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:04.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:04.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:04.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:04.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:04.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:00 2026-02-22T23:02:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:02:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:03' 2026-02-22T23:02:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:02:04.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:02:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:04.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:02:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:04.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:04.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:00' 2026-02-22T23:02:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:02:00' 2026-02-22T23:02:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:04.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:00' 2026-02-22T23:02:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:04.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:02:00cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:02:03' 2026-02-22T23:02:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:00' 2026-02-22T23:02:04.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:02:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:02:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:02:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:02:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:02:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:02:06.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:02:06.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:06.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:02:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:02:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:02:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:02:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:02:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:02:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:02:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:02:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:02:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:02:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:02:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:02:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:02:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:02:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:02:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:02:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:02:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:02:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:02:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:02:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:02:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:02:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:02:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:02:07.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:02:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:02:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:02:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:02:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:02:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:02:07.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:02:10.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 47 2026-02-22T23:02:10.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:02:10.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:02:10.942 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:10.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:10.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:10.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:02:10.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:02:10.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:02:10.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:02:10.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:02:10.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:02:10.944 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:02:10.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:02:10.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:02:10.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:10.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:10.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:10.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:02:10.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:02:10.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:02:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:02:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=47 2026-02-22T23:02:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:02:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:02:10.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:02:11.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:02:11.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:11.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:11.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:11.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:02:11.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:02:11.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:02:11.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=47 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 47 = 47 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:11.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:02:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:02:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:02:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:02:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:02:12.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:08 2026-02-22T23:02:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:02:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:02:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801325,"replay_state":"idle"} 2026-02-22T23:02:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:05 2026-02-22T23:02:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:02:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 47 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.e9a22529-7e1a-4cfe-afa0-0bb8e498d0c5 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:02:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:02:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:02:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:12.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:02:12.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:12.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:02:12.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:12.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:02:12.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:12.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:12.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:12.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:08' 2026-02-22T23:02:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:02:08' 2026-02-22T23:02:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:02:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:08' 2026-02-22T23:02:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:12.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:02:08cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801325,"replay_state":"idle"}2026-02-22 23:02:05' 2026-02-22T23:02:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:08' 2026-02-22T23:02:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:02:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:02:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:02:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:02:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:02:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:02:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:02:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:02:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:13.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801325,"replay_state":"idle"} 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:05 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:08' 2026-02-22T23:02:13.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:02:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:02:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801325,"replay_state":"idle"} 2026-02-22T23:02:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:02:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:02:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:05' 2026-02-22T23:02:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:02:05' 2026-02-22T23:02:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:02:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:05' 2026-02-22T23:02:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:13.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801325,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:02:05cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:02:08' 2026-02-22T23:02:13.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:13.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:05' 2026-02-22T23:02:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:02:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:02:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:02:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:02:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:02:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:13.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:02:13.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:02:13.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:02:13.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:13.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:13.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:13.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:02:13.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:02:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:02:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:02:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:02:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:02:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:02:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:02:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:02:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:02:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:02:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:02:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:02:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:02:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:02:14.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:02:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:02:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:02:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:02:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:02:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:02:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:02:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:02:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:02:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:02:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:02:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:02:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:02:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:02:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:02:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:02:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:02:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:02:15.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:02:15.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:02:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:02:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:02:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:02:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:15.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:15.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:15.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:02:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:02:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:02:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:02:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:13 2026-02-22T23:02:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:02:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:15' 2026-02-22T23:02:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:02:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:02:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:02:18.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:02:18.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:18.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:18.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:18.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:18.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:18.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:02:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:02:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:02:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:13 2026-02-22T23:02:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:18.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:02:18.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:18.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:18.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:15' 2026-02-22T23:02:18.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:02:18.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:02:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:02:22.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:02:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:22.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:22.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:02:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:18 2026-02-22T23:02:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:02:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:20' 2026-02-22T23:02:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:02:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:02:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:02:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:22.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:18' 2026-02-22T23:02:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:02:18' 2026-02-22T23:02:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:22.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:18' 2026-02-22T23:02:22.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:22.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:22.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:02:18cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:02:20' 2026-02-22T23:02:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:18' 2026-02-22T23:02:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:02:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:02:22.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:02:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:22.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:22.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:22.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:22.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:02:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:02:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:20 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:18' 2026-02-22T23:02:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:02:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:02:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:02:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:23.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:23.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:23.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:20' 2026-02-22T23:02:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:02:20' 2026-02-22T23:02:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:20' 2026-02-22T23:02:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:02:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:02:18' 2026-02-22T23:02:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:20' 2026-02-22T23:02:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:02:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:02:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:02:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:02:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:02:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:02:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:02:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:02:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:02:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:02:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:02:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:02:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:02:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:02:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:02:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:02:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:02:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:02:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:02:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:02:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:02:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:02:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:02:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:02:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:02:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:02:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:02:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:02:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:02:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:02:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:02:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:02:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:02:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:02:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:02:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:02:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:02:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:02:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:02:28.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:02:28.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:28.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:28.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:28.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:02:28.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:02:28.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:02:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:02:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:02:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:02:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:02:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:02:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:02:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:02:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:02:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:02:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:02:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:02:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:02:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:02:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:02:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:02:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:02:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:02:28.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:28.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:28.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:28.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:02:28.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:02:28.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:02:28.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:02:28.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:02:28.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:02:28.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:28.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:02:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:02:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:02:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:02:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:02:32.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:02:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:02:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:02:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:02:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:02:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:02:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:02:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:02:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:02:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:02:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:02:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:02:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:02:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:02:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:02:32.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:02:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:02:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:02:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:02:32.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:02:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:02:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:02:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:02:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:02:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:32.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:02:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 50 2026-02-22T23:02:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:02:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:02:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:02:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:02:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:02:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:02:35.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:02:35.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:02:35.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:02:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:02:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:02:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:02:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:02:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:02:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:02:36.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=50 2026-02-22T23:02:36.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:02:36.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:02:36.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:02:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:02:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:02:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:02:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:02:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:02:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=50 2026-02-22T23:02:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 50 = 50 2026-02-22T23:02:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:02:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:02:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:02:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:02:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:02:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:02:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:02:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:35 2026-02-22T23:02:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:02:37.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:02:37.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801343,"replay_state":"idle"} 2026-02-22T23:02:37.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:28 2026-02-22T23:02:37.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:02:37.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 50 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.6e72af5c-0d5b-4af1-99b3-42352973ce31 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:02:37.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:02:37.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:02:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:02:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:02:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:02:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:37.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:35' 2026-02-22T23:02:37.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:02:35' 2026-02-22T23:02:37.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:37.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:37.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:37.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:02:37.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:35' 2026-02-22T23:02:37.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:37.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:37.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:02:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801343,"replay_state":"idle"}2026-02-22 23:02:28' 2026-02-22T23:02:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:37.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:35' 2026-02-22T23:02:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:37.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:02:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:02:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:02:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:02:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:02:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:02:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:37.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:02:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:02:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:02:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:02:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801343,"replay_state":"idle"} 2026-02-22T23:02:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:28 2026-02-22T23:02:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:02:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:02:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:02:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:35' 2026-02-22T23:02:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:02:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:02:38.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:38.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801343,"replay_state":"idle"} 2026-02-22T23:02:38.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:38.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:02:38.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:38.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:02:38.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:38.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:28' 2026-02-22T23:02:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:02:28' 2026-02-22T23:02:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:38.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:02:38.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:28' 2026-02-22T23:02:38.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:38.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:38.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801343,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:02:28cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:02:35' 2026-02-22T23:02:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:28' 2026-02-22T23:02:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:02:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:02:38.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:02:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:02:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:02:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:02:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:02:38.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:02:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:02:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:02:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:02:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:02:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:02:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:02:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:02:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:02:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:02:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:02:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:02:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:02:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:02:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:02:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:02:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:02:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:02:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:02:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:02:39.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:02:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:02:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:02:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:02:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:02:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:02:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:02:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:02:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:02:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:02:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:02:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:02:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:02:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:02:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:02:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:02:42.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:02:42.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:42.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:02:42.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:02:42.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:02:42.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:42.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:02:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:02:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:02:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:02:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:02:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:02:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:02:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:02:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:02:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:02:42.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:02:42.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:02:42.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:02:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:40 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:40' 2026-02-22T23:02:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:02:43.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:02:43.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:43.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image demoted 2026-02-22T23:02:43.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:43.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:43.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:40' 2026-02-22T23:02:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:02:40' 2026-02-22T23:02:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:40' 2026-02-22T23:02:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image demoted43224326mirror.0trial0822026-02-22 23:02:40cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:02:40' 2026-02-22T23:02:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:40' 2026-02-22T23:02:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:02:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:02:43.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:43.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:43.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:43.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:02:43.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:02:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:44.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:44.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:02:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:40 2026-02-22T23:02:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:02:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:43' 2026-02-22T23:02:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:02:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:02:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:02:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:40' 2026-02-22T23:02:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:02:40' 2026-02-22T23:02:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:40' 2026-02-22T23:02:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:44.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:02:40cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:02:43' 2026-02-22T23:02:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:40' 2026-02-22T23:02:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:44.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:02:44.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:02:44.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:02:44.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:02:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:02:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:02:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:02:45.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:02:45.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:46.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:02:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:02:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:02:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:02:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:02:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:02:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:02:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:02:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:02:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:02:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:02:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:02:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:02:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:02:46.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:02:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:46.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:02:47.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:02:47.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:02:47.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:02:47.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:02:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:02:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 53 2026-02-22T23:02:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:02:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:02:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:02:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:02:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:02:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:02:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:02:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:02:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:02:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:02:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:02:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:02:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:02:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:02:50.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:02:50.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=53 2026-02-22T23:02:50.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:02:50.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:02:50.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:02:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:02:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:02:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:02:51.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:02:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:02:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=53 2026-02-22T23:02:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 53 = 53 2026-02-22T23:02:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:02:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:02:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:02:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:02:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:52.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:52.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:02:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:02:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:02:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:48 2026-02-22T23:02:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:02:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:02:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801364,"replay_state":"idle"} 2026-02-22T23:02:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:45 2026-02-22T23:02:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:02:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 53 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.243e0fc8-4860-4d99-8e85-3aa70fc25327 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:02:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:02:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:02:52.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:52.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:02:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:02:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:02:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:52.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:52.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:52.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:48' 2026-02-22T23:02:52.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:02:48' 2026-02-22T23:02:52.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:52.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:52.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:02:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:48' 2026-02-22T23:02:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:02:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801364,"replay_state":"idle"}2026-02-22 23:02:45' 2026-02-22T23:02:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:48' 2026-02-22T23:02:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:52.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:02:52.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:02:52.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:02:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:02:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:02:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:02:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:02:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:02:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:02:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801364,"replay_state":"idle"} 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:45 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:48' 2026-02-22T23:02:53.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:02:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:02:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:02:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801364,"replay_state":"idle"} 2026-02-22T23:02:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:02:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:02:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:02:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:02:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:02:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:02:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:02:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:02:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:45' 2026-02-22T23:02:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:02:45' 2026-02-22T23:02:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:02:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:45' 2026-02-22T23:02:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:02:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801364,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:02:45cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:02:48' 2026-02-22T23:02:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:02:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:02:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:45' 2026-02-22T23:02:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:02:53.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:02:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:02:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:02:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:02:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:02:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:02:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:02:53.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:02:53.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:53.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:53.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:53.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:02:53.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:53.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:02:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:02:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:02:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:02:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:02:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:02:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:02:54.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:02:54.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:02:54.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:02:54.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:54.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:02:54.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:02:54.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:02:54.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:54.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:02:54.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:02:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:02:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:02:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:02:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:02:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:02:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:02:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:02:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:02:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:02:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:02:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:02:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:02:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:02:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:02:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:02:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:02:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:02:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:02:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:02:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:02:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:02:54.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:54.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:54.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:02:54.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:02:55.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:02:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:02:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:02:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:02:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:02:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:02:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:02:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:02:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:02:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:56.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:02:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:02:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:02:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:53 2026-02-22T23:02:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:02:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:55' 2026-02-22T23:02:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:02:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:02:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:02:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:02:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:02:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:02:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:02:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:02:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:02:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:02:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:02:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:02:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:02:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:02:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:02:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:02:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:02:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:02:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:53 2026-02-22T23:02:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:02:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:02:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:02:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:02:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:55' 2026-02-22T23:02:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:02:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:02:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:02:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:03:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:03:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:03:02.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:02.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:02.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:58 2026-02-22T23:03:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:03:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:00' 2026-02-22T23:03:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:03:02.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:03:02.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:02.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:03:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:58' 2026-02-22T23:03:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:02:58' 2026-02-22T23:03:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:02:58' 2026-02-22T23:03:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:02:58cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:03:00' 2026-02-22T23:03:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:02.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:02:58' 2026-02-22T23:03:02.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:02.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:03:02.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:02.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:03:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:03:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:03:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:03:03.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:00 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:02:58' 2026-02-22T23:03:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:03:03.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:03:03.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:03:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:00' 2026-02-22T23:03:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:03:00' 2026-02-22T23:03:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:00' 2026-02-22T23:03:03.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:03.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:03.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:03:00cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:02:58' 2026-02-22T23:03:03.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:03.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:03.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:00' 2026-02-22T23:03:03.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:03.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:03:03.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:03.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:03:03.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:03.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:03.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:03.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:03:03.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:03.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:03.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:03.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:03:03.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:03:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:03:04.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:03:04.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:04.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:04.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:04.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:03:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:03:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:03:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:03:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:03:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:03:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:03:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:03:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:03:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:03:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:03:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:03:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:03:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:03:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:03:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:03:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:03:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:03:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:06.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:03:06.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:03:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:08.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:03:08.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:03:08.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:08.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:08.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:08.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:03:08.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:03:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:03:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:08.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:03:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:03:08.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:08.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:08.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:08.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:03:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:03:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:03:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:03:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:03:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:03:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:03:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:03:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:03:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:03:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:03:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:03:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:03:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:03:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:03:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:12.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:03:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:03:12.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:03:12.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:12.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:03:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:03:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:03:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:03:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:03:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:03:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:03:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:03:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:03:14.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 56 2026-02-22T23:03:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:03:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:03:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:03:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:03:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:03:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:03:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:03:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:03:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:03:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:03:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:03:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=56 2026-02-22T23:03:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:03:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:03:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:03:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:03:15.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:03:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:03:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=56 2026-02-22T23:03:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 56 = 56 2026-02-22T23:03:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:03:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:15.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:15.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:03:15.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:15.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:15.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:15.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:03:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:03:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:03:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:03:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:03:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:15 2026-02-22T23:03:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:03:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:03:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801383,"replay_state":"idle"} 2026-02-22T23:03:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:08 2026-02-22T23:03:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:03:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 56 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.a16ade1a-1f01-4ee9-bfa5-e715a9e3b83e (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:03:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:03:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:03:16.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:16.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:03:16.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:16.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:03:16.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:16.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:03:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:15' 2026-02-22T23:03:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:03:15' 2026-02-22T23:03:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:03:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:15' 2026-02-22T23:03:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:16.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:03:15cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801383,"replay_state":"idle"}2026-02-22 23:03:08' 2026-02-22T23:03:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:16.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:15' 2026-02-22T23:03:16.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:03:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:03:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:03:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:03:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:03:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:03:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:03:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:03:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:03:17.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:17.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:03:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801383,"replay_state":"idle"} 2026-02-22T23:03:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:08 2026-02-22T23:03:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:03:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:03:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:03:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:15' 2026-02-22T23:03:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:03:17.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:03:17.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:17.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801383,"replay_state":"idle"} 2026-02-22T23:03:17.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:17.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:03:17.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:17.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:03:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:08' 2026-02-22T23:03:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:03:08' 2026-02-22T23:03:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:17.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:03:17.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:08' 2026-02-22T23:03:17.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:17.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:17.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:17.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801383,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:03:08cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:03:15' 2026-02-22T23:03:17.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:17.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:17.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:08' 2026-02-22T23:03:17.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:17.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:03:17.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:03:17.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:03:17.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:03:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:03:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:03:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:03:17.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:03:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:03:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:03:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:03:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:03:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:03:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:03:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:03:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:03:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:03:18.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:18.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:18.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:03:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:19.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:03:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:03:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:03:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:03:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:03:21.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:03:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:03:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:03:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:21.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:21.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:21.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:03:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:03:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:03:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:03:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:03:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:21.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:03:21.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:21.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:03:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:21.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:03:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:03:22.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:22.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:22.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:22.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:03:22.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:22.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:19 2026-02-22T23:03:22.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:22.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:03:22.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:22.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:22.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:20' 2026-02-22T23:03:22.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:03:22.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:03:22.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:22.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image demoted 2026-02-22T23:03:22.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:19' 2026-02-22T23:03:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:03:19' 2026-02-22T23:03:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:19' 2026-02-22T23:03:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:22.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image demoted43224326mirror.0trial0822026-02-22 23:03:19cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:03:20' 2026-02-22T23:03:22.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:22.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:19' 2026-02-22T23:03:22.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:22.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:03:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:22.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:03:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:03:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:03:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:03:23.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:23.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:20 2026-02-22T23:03:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:03:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:03:23.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:19' 2026-02-22T23:03:23.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:03:23.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:03:23.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:23.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:03:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:23.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:23.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:23.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:23.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:20' 2026-02-22T23:03:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:03:20' 2026-02-22T23:03:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:23.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:23.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:20' 2026-02-22T23:03:23.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:23.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:23.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:03:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image demoted2026-02-22 23:03:19' 2026-02-22T23:03:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:20' 2026-02-22T23:03:23.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:23.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:03:23.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:23.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:03:23.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:03:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:23.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:23.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:03:23.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:03:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:03:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:03:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:03:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:03:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:03:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:25.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:03:25.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:03:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:25.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:25.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:25.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:03:25.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:25.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:25.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:03:25.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:03:25.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:03:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:03:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:03:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:03:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:26.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:26.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:03:26.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:03:26.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:26.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:03:26.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:03:26.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:03:26.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:26.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:03:26.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:03:26.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:26.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:26.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:26.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:03:26.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:03:26.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:03:26.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:26.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:26.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:26.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:03:29.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 59 2026-02-22T23:03:29.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:03:29.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:03:29.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:29.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:29.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:29.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:03:29.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:03:29.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:03:29.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:29.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:03:29.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:03:29.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:03:29.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:29.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:29.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:29.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:29.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:29.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:29.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:03:29.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:03:29.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:03:29.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=59 2026-02-22T23:03:29.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:03:29.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:03:29.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:03:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:03:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:03:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:03:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=59 2026-02-22T23:03:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 59 = 59 2026-02-22T23:03:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:03:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:03:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:03:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:03:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:03:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:03:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:03:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:28 2026-02-22T23:03:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:03:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:03:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801403,"replay_state":"idle"} 2026-02-22T23:03:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:25 2026-02-22T23:03:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:03:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 59 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.0b1a6e08-d2af-4902-843e-96a9d3e082db (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:03:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:03:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:03:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:03:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:03:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:03:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:28' 2026-02-22T23:03:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:03:28' 2026-02-22T23:03:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:03:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:28' 2026-02-22T23:03:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:03:28cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801403,"replay_state":"idle"}2026-02-22 23:03:25' 2026-02-22T23:03:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:28' 2026-02-22T23:03:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:31.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:03:31.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:03:31.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:03:31.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:03:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:03:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:03:31.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:31.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:31.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:31.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:32.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:03:32.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:32.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:32.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:32.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:03:32.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:32.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:32.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:32.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:32.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:03:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:03:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801403,"replay_state":"idle"} 2026-02-22T23:03:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:25 2026-02-22T23:03:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:03:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:03:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:03:32.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:28' 2026-02-22T23:03:32.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:03:32.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:03:32.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:32.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801403,"replay_state":"idle"} 2026-02-22T23:03:32.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:32.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:03:32.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:32.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:03:32.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:32.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:32.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:32.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:25' 2026-02-22T23:03:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:03:25' 2026-02-22T23:03:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:03:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:25' 2026-02-22T23:03:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801403,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:03:25cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:03:28' 2026-02-22T23:03:32.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:32.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:32.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:25' 2026-02-22T23:03:32.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:03:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:03:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:03:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:03:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:32.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:03:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:03:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:32.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:03:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:03:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:03:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:33.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:33.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:03:33.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:03:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:03:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:03:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:03:33.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:03:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:03:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:03:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:03:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:03:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:03:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:03:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:03:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:03:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:03:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:03:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:03:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:03:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:34.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:03:34.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:03:34.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:03:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:35.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:03:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:35.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:03:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:33 2026-02-22T23:03:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:03:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:03:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:33' 2026-02-22T23:03:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:03:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:03:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:03:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:33' 2026-02-22T23:03:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:03:33' 2026-02-22T23:03:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:33' 2026-02-22T23:03:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:03:33cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:03:33' 2026-02-22T23:03:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:33' 2026-02-22T23:03:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:03:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:03:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:03:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:35 2026-02-22T23:03:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:03:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:33' 2026-02-22T23:03:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:03:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:03:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:03:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:35' 2026-02-22T23:03:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:03:35' 2026-02-22T23:03:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:35' 2026-02-22T23:03:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:36.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:03:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:03:33' 2026-02-22T23:03:36.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:36.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:35' 2026-02-22T23:03:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:03:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:03:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:03:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:03:37.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:03:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:03:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:03:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:03:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:03:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:03:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:03:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:03:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:03:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:39.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:03:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:03:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:03:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:03:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:03:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:03:39.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:03:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:39.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:03:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:39.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:03:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:03:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:03:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:03:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:03:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:03:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:03:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:03:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:03:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:03:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:03:43.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 63 2026-02-22T23:03:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:03:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:03:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:03:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:03:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:03:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:03:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:03:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:03:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:03:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:03:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:03:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=63 2026-02-22T23:03:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:03:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:03:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:03:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:03:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:03:43.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:03:43.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=63 2026-02-22T23:03:43.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 63 = 63 2026-02-22T23:03:43.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:43.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:03:43.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:43.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:43.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:43.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:03:43.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:43.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:43.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:43.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:03:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:03:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:44.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:03:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:03:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:40 2026-02-22T23:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801416,"replay_state":"idle"} 2026-02-22T23:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:38 2026-02-22T23:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 63 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.4c3f1bbe-238b-467c-826a-9cba71fe1fd2 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:03:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:03:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:03:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:03:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:03:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:03:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:44.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:44.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:40' 2026-02-22T23:03:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:03:40' 2026-02-22T23:03:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:03:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:40' 2026-02-22T23:03:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:03:40cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801416,"replay_state":"idle"}2026-02-22 23:03:38' 2026-02-22T23:03:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:40' 2026-02-22T23:03:44.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:03:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:03:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:03:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:03:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:03:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:03:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:03:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:03:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:03:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801416,"replay_state":"idle"} 2026-02-22T23:03:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:38 2026-02-22T23:03:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:03:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:03:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:03:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:40' 2026-02-22T23:03:45.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:03:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:03:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801416,"replay_state":"idle"} 2026-02-22T23:03:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:03:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:03:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:38' 2026-02-22T23:03:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:03:38' 2026-02-22T23:03:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:03:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:38' 2026-02-22T23:03:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801416,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:03:38cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:03:40' 2026-02-22T23:03:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:45.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:38' 2026-02-22T23:03:45.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:03:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:03:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:03:45.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:03:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:03:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:03:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:03:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:03:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:03:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:03:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:03:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:03:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:03:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:03:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:03:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:03:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:03:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:03:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:03:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:03:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:03:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:03:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:03:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:03:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:03:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:03:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:03:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:03:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:03:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:47.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:47.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:03:47.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:03:47.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:47.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:03:47.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:03:47.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:03:47.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:47.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:03:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:03:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:03:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:03:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:03:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:03:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:03:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:03:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:03:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:03:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:03:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:03:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:03:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:03:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:03:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:49.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:03:49.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:03:49.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:03:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:03:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:49.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:49.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:49.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:49.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:49.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:03:50.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:03:50.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:48 2026-02-22T23:03:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:03:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:03:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:03:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:45' 2026-02-22T23:03:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:03:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:03:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:03:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:48' 2026-02-22T23:03:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:03:48' 2026-02-22T23:03:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:48' 2026-02-22T23:03:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:03:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:03:45' 2026-02-22T23:03:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:48' 2026-02-22T23:03:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:03:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:03:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:51.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:03:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:03:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:03:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:03:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:03:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:50 2026-02-22T23:03:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:03:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:03:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:03:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:03:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:48' 2026-02-22T23:03:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:03:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:03:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:03:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:03:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:03:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:03:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:03:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:03:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:03:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:03:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:50' 2026-02-22T23:03:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:03:50' 2026-02-22T23:03:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:03:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:50' 2026-02-22T23:03:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:03:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:03:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:03:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:03:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:03:48' 2026-02-22T23:03:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:03:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:03:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:50' 2026-02-22T23:03:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:03:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:03:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:03:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:03:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:03:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:03:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:03:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:03:52.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:03:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:03:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:03:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:03:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:03:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:03:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:03:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:03:53.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:53.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:53.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:53.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:03:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:03:53.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:03:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:03:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:03:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:03:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:03:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:03:53.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:03:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:03:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:03:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:03:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:03:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:03:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:54.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:54.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:03:54.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:03:54.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:54.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:03:54.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:54.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:03:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:03:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:03:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:03:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:03:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:03:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:03:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:03:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:03:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:03:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:03:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:56.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:03:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:03:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:03:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:03:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:03:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:03:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:03:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:03:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:03:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:03:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:03:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:03:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:03:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:03:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:03:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:03:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:03:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:03:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:03:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:03:56.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:03:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:03:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:03:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:03:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:03:56.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:03:56.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:03:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:03:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:03:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:03:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:03:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:03:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:03:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:03:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:03:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:03:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:03:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:03:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:56.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:03:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 65 2026-02-22T23:03:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:03:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:03:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:03:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:03:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:03:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:03:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:03:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:03:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:03:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:03:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:03:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:03:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=65 2026-02-22T23:03:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:03:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:03:59.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:03:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:03:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:03:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:03:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:03:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:03:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=65 2026-02-22T23:03:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 65 = 65 2026-02-22T23:03:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:03:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:03:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:03:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:03:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:03:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:03:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:03:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:03:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:03:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:04:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:04:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:58 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801431,"replay_state":"idle"} 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:55 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 64 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.471b84ee-f480-4dab-993b-bda5d1f6080f (peer_uuids:[]) 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 65 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.581cb220-6e77-43d6-8424-b412e494ab2d (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:04:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:04:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:04:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:04:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:58' 2026-02-22T23:04:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:03:58' 2026-02-22T23:04:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:58' 2026-02-22T23:04:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:03:58cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801431,"replay_state":"idle"}2026-02-22 23:03:55' 2026-02-22T23:04:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:58' 2026-02-22T23:04:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:00.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:04:00.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:04:00.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:04:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:04:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:04:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801431,"replay_state":"idle"} 2026-02-22T23:04:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:55 2026-02-22T23:04:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:04:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:04:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:04:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:03:58' 2026-02-22T23:04:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:04:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:04:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801431,"replay_state":"idle"} 2026-02-22T23:04:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:04:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:04:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:01.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:01.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:55' 2026-02-22T23:04:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:03:55' 2026-02-22T23:04:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:03:55' 2026-02-22T23:04:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:01.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801431,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:03:55cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:03:58' 2026-02-22T23:04:01.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:01.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:03:55' 2026-02-22T23:04:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:04:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:04:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:04:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:04:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:04:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:04:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:04:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:04:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:04:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:02.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:02.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:02.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:04:02.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:04:02.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:02.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:04:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:04:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:04:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:04:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:04:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:04:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:04:03.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:04:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:03.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:04:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:04:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:04:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:04:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:04:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:04:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:04:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:04:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:04:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:03 2026-02-22T23:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:03' 2026-02-22T23:04:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:04:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:04:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:04:05.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:05.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:05.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:03' 2026-02-22T23:04:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:04:03' 2026-02-22T23:04:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:03' 2026-02-22T23:04:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:04:03cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:04:03' 2026-02-22T23:04:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:03' 2026-02-22T23:04:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:05.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:04:05.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:05.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:04:05.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:05.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:05.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:05.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:04:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:04:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:04:06.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:05 2026-02-22T23:04:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:04:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:03' 2026-02-22T23:04:06.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:04:06.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:04:06.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:04:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:05' 2026-02-22T23:04:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:04:05' 2026-02-22T23:04:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:05' 2026-02-22T23:04:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:04:05cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:04:03' 2026-02-22T23:04:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:05' 2026-02-22T23:04:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:04:06.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:06.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:04:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:04:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:04:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:04:07.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:04:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:04:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:04:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:04:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:08.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:04:09.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:04:09.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:09.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:09.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:09.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:04:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:04:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:04:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:04:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:04:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:04:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:04:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:04:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:04:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:09.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:04:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:04:09.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:04:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:04:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:04:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:04:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:04:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:04:09.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:04:09.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:04:09.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:09.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:09.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:09.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:04:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 69 2026-02-22T23:04:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:04:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:04:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:04:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:04:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:04:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:04:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:04:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:04:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:04:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:04:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:04:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=69 2026-02-22T23:04:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:04:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:04:13.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:04:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:04:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:04:13.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:04:13.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=69 2026-02-22T23:04:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 69 = 69 2026-02-22T23:04:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:04:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:04:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:04:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:04:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:04:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:10 2026-02-22T23:04:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:04:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:04:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801446,"replay_state":"idle"} 2026-02-22T23:04:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:08 2026-02-22T23:04:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:04:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 69 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.8ddba7ff-8868-4292-8351-7573b853990b (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:04:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:04:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:04:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:04:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:04:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:04:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:14.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:10' 2026-02-22T23:04:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:04:10' 2026-02-22T23:04:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:10' 2026-02-22T23:04:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:04:10cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801446,"replay_state":"idle"}2026-02-22 23:04:08' 2026-02-22T23:04:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:10' 2026-02-22T23:04:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:04:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:04:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:04:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:04:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:04:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:04:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:04:15.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801446,"replay_state":"idle"} 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:08 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:10' 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:04:15.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:04:15.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:15.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801446,"replay_state":"idle"} 2026-02-22T23:04:15.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:15.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:04:15.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:15.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:04:15.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:15.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:15.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:15.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:08' 2026-02-22T23:04:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:04:08' 2026-02-22T23:04:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:08' 2026-02-22T23:04:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:15.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801446,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:04:08cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:04:10' 2026-02-22T23:04:15.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:15.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:08' 2026-02-22T23:04:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:04:15.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:04:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:04:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:04:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:04:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:04:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:15.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:15.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:04:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:17.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:04:17.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:04:17.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:04:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:04:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:04:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:04:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:17.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:17.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:04:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:17.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:17.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:04:17.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:04:17.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:04:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:04:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:04:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:17.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:04:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:04:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:04:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:04:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:04:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:04:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:04:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:04:19.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:04:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:19.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:19.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:04:19.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:04:19.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:04:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:20.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:04:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:18 2026-02-22T23:04:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:20.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:04:20.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:04:20.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:04:20.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:15' 2026-02-22T23:04:20.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:04:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:04:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:04:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:18' 2026-02-22T23:04:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:04:18' 2026-02-22T23:04:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:18' 2026-02-22T23:04:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:04:18cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:04:15' 2026-02-22T23:04:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:18' 2026-02-22T23:04:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:04:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:04:20.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:20.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:04:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:04:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:20 2026-02-22T23:04:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:04:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:18' 2026-02-22T23:04:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:04:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:04:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:04:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:21.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:21.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:20' 2026-02-22T23:04:21.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:04:20' 2026-02-22T23:04:21.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:21.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:21.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:21.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:20' 2026-02-22T23:04:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:04:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:04:18' 2026-02-22T23:04:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:20' 2026-02-22T23:04:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:21.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:04:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:04:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:04:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:04:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:04:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:04:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:04:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:04:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:04:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:04:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:04:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:23.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:23.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:04:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:04:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:04:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:24.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:04:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:04:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:04:24.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:24.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:04:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:04:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:04:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:26.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:26.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:04:26.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:04:26.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:04:28.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 71 2026-02-22T23:04:28.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:04:28.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:04:28.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:28.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:28.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:28.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:04:28.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:04:28.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:04:28.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:04:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:04:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:04:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:04:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:04:28.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:04:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=71 2026-02-22T23:04:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:04:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:04:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:04:29.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:29.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:29.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:29.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:29.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:29.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:04:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:04:29.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:04:29.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=71 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 71 = 71 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:30.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:04:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:28 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801461,"replay_state":"idle"} 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:25 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 70 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.c2fc05ae-abed-4720-8313-944b582b2da7 (peer_uuids:[]) 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 71 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.d417ffa0-527c-4444-8bc2-ed51869b3ad2 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:04:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:04:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:04:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:04:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:04:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:30.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:04:30.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:28' 2026-02-22T23:04:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:04:28' 2026-02-22T23:04:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:28' 2026-02-22T23:04:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:04:28cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801461,"replay_state":"idle"}2026-02-22 23:04:25' 2026-02-22T23:04:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:28' 2026-02-22T23:04:30.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:04:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:04:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:04:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:04:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:04:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801461,"replay_state":"idle"} 2026-02-22T23:04:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:25 2026-02-22T23:04:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:04:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:04:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:04:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:28' 2026-02-22T23:04:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:04:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:04:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801461,"replay_state":"idle"} 2026-02-22T23:04:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:04:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:04:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:25' 2026-02-22T23:04:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:04:25' 2026-02-22T23:04:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:25' 2026-02-22T23:04:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801461,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:04:25cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:04:28' 2026-02-22T23:04:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:31.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:25' 2026-02-22T23:04:31.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:04:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:04:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:04:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:04:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:04:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:04:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:04:32.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:04:32.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:32.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:32.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:32.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:04:32.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:32.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:32.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:32.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:04:32.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:04:32.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:32.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:04:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:04:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:33.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:04:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:04:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:04:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:04:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:04:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:04:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:04:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:04:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:04:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:33 2026-02-22T23:04:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:04:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:04:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:33' 2026-02-22T23:04:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:04:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:04:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:04:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:33' 2026-02-22T23:04:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:04:33' 2026-02-22T23:04:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:33' 2026-02-22T23:04:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:04:33cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:04:33' 2026-02-22T23:04:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:33' 2026-02-22T23:04:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:04:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:04:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:04:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:04:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:36.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:36.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:36.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:36.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:36.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:36.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:36.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:36.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:04:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:35 2026-02-22T23:04:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:04:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:33' 2026-02-22T23:04:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:04:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:04:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:36.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:04:36.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:36.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:36.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:36.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:36.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:36.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:36.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:36.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:35' 2026-02-22T23:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:04:35' 2026-02-22T23:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:35' 2026-02-22T23:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:04:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:04:33' 2026-02-22T23:04:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:36.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:35' 2026-02-22T23:04:36.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:04:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:04:36.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:36.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:36.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:04:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:04:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:04:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:04:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:04:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:04:38.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:38.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:38.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:38.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:04:38.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:04:38.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:38.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:04:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:04:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:04:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:04:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:39.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:04:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:04:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:04:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:04:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:04:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:39.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:04:39.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:04:39.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:39.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:04:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:04:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:39.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:04:43.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 75 2026-02-22T23:04:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:04:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:04:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:43.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:04:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:04:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:04:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:04:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:04:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:04:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:04:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:04:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:04:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=75 2026-02-22T23:04:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:04:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:04:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:04:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:04:43.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:04:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:04:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=75 2026-02-22T23:04:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 75 = 75 2026-02-22T23:04:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:04:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:44.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:04:44.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:44.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:44.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:44.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:44.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:40 2026-02-22T23:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801476,"replay_state":"idle"} 2026-02-22T23:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:38 2026-02-22T23:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 75 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.8bce1be2-30ab-41cd-afa2-6883acdee177 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:04:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:04:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:04:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:04:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:40' 2026-02-22T23:04:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:04:40' 2026-02-22T23:04:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:40' 2026-02-22T23:04:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:44.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:44.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:04:40cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801476,"replay_state":"idle"}2026-02-22 23:04:38' 2026-02-22T23:04:44.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:44.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:40' 2026-02-22T23:04:44.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:44.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:04:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:04:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:04:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:04:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:45.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:45.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:04:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:04:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801476,"replay_state":"idle"} 2026-02-22T23:04:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:38 2026-02-22T23:04:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:04:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:04:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:04:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:40' 2026-02-22T23:04:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:04:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:04:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801476,"replay_state":"idle"} 2026-02-22T23:04:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:04:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:45.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:04:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:38' 2026-02-22T23:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:04:38' 2026-02-22T23:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:38' 2026-02-22T23:04:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:45.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801476,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:04:38cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:04:40' 2026-02-22T23:04:45.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:45.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:38' 2026-02-22T23:04:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:04:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:04:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:04:45.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:04:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:04:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:04:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:04:46.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:04:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:04:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:04:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:04:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:04:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:04:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:04:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:04:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:04:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:04:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:04:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:04:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:04:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:47.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:04:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:04:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:04:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:04:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:04:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:04:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:04:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:04:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:04:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:04:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:04:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:04:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:04:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:04:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:04:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:04:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:04:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:04:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:04:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:04:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:04:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:04:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:04:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:04:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:04:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:04:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:04:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:04:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:04:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:04:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:04:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:48 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:45' 2026-02-22T23:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:04:50.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:04:50.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:50.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:04:50.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:50.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:48' 2026-02-22T23:04:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:04:48' 2026-02-22T23:04:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:48' 2026-02-22T23:04:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:04:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:04:45' 2026-02-22T23:04:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:48' 2026-02-22T23:04:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:04:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:04:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:04:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:04:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:51.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:04:51.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:04:51.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:04:51.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:51.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:51.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:04:51.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:50 2026-02-22T23:04:51.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:04:51.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:04:51.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:04:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:04:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:48' 2026-02-22T23:04:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:04:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:04:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:04:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:04:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:04:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:04:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:04:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:04:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:04:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:04:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:50' 2026-02-22T23:04:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:04:50' 2026-02-22T23:04:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:04:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:50' 2026-02-22T23:04:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:04:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:04:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:04:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:04:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:04:48' 2026-02-22T23:04:51.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:04:51.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:04:51.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:50' 2026-02-22T23:04:51.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:04:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:04:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:04:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:04:51.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:04:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:04:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:04:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:04:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:04:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:04:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:04:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:04:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:04:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:04:53.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:04:53.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:53.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:53.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:53.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:04:53.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:53.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:53.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:04:53.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:04:53.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:04:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:04:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:04:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:04:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:04:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:04:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:54.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:54.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:04:54.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:04:54.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:54.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:04:54.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:54.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:04:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:04:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:04:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:04:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:04:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:04:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:04:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:04:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:04:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:04:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:04:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:04:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:04:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:04:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:04:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:04:56.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:56.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:56.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:04:56.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:04:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:04:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:04:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:04:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:04:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:04:58.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 77 2026-02-22T23:04:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:04:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:04:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:04:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:04:58.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:04:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:04:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:04:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:04:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:04:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:04:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:04:59.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:04:59.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=77 2026-02-22T23:04:59.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:04:59.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:04:59.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:04:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:04:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:04:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:04:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:04:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:04:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=77 2026-02-22T23:04:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 77 = 77 2026-02-22T23:04:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:04:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:04:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:04:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:04:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:04:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:04:59.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:04:59.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:04:59.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:04:59.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:05:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:05:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:05:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:05:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:58 2026-02-22T23:05:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:05:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:05:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801491,"replay_state":"idle"} 2026-02-22T23:05:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:55 2026-02-22T23:05:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:05:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 76 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.4820f0c3-ab61-421c-9f79-4ec9417ecfd5 (peer_uuids:[]) 2026-02-22T23:05:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 77 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.5ad6133c-5685-4369-88d4-8f2083f04dca (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:05:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:05:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:05:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:05:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:05:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:05:00.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:00.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:58' 2026-02-22T23:05:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:04:58' 2026-02-22T23:05:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:58' 2026-02-22T23:05:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:00.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:04:58cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801491,"replay_state":"idle"}2026-02-22 23:04:55' 2026-02-22T23:05:00.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:00.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:58' 2026-02-22T23:05:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:05:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:05:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:05:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:05:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:05:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:01.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:05:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:01.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:05:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:05:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801491,"replay_state":"idle"} 2026-02-22T23:05:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:55 2026-02-22T23:05:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:05:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:05:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:05:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:04:58' 2026-02-22T23:05:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:05:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:05:01.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:01.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801491,"replay_state":"idle"} 2026-02-22T23:05:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:05:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:05:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:55' 2026-02-22T23:05:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:04:55' 2026-02-22T23:05:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:04:55' 2026-02-22T23:05:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801491,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:04:55cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:04:58' 2026-02-22T23:05:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:04:55' 2026-02-22T23:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:01.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:05:01.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:05:01.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:05:01.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:05:01.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:01.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:01.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:05:01.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:01.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:01.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:01.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:05:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:05:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:05:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:05:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:05:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:05:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:05:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:05:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:05:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:05:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:05:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:05:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:05:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:05:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:05:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:05:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:05:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:05:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:05:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:03 2026-02-22T23:05:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:05:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:05:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:02' 2026-02-22T23:05:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:05:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:05:05.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:05.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:05:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:03' 2026-02-22T23:05:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:05:03' 2026-02-22T23:05:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:03' 2026-02-22T23:05:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:05:03cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:05:02' 2026-02-22T23:05:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:05.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:03' 2026-02-22T23:05:05.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:05.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:05:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:05:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:05:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:05:06.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:05:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:05 2026-02-22T23:05:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:05:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:03' 2026-02-22T23:05:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:05:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:05:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:05:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:05' 2026-02-22T23:05:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:05:05' 2026-02-22T23:05:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:05' 2026-02-22T23:05:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:06.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:05:05cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:05:03' 2026-02-22T23:05:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:06.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:05' 2026-02-22T23:05:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:05:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:05:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:05:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:05:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:05:07.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:05:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:05:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:05:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:05:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:05:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:05:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:05:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:09.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:05:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:05:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:05:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:05:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:05:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:05:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:05:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:05:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:05:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:05:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:05:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:05:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:05:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:05:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:05:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:05:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:05:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:05:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:05:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:05:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:05:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:09.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:05:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 81 2026-02-22T23:05:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:05:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:05:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:05:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:05:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:05:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:05:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:05:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:05:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:05:13.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:05:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:05:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=81 2026-02-22T23:05:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:05:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:05:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:05:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:05:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:05:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:05:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=81 2026-02-22T23:05:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 81 = 81 2026-02-22T23:05:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:05:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:05:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:14.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:10 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801506,"replay_state":"idle"} 2026-02-22T23:05:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:08 2026-02-22T23:05:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:05:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 81 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.8bd72137-5753-4c81-82af-e2dc942ae24a (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:05:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:05:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:05:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:05:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:05:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:05:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:14.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:14.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:14.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:10' 2026-02-22T23:05:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:05:10' 2026-02-22T23:05:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:10' 2026-02-22T23:05:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:05:10cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801506,"replay_state":"idle"}2026-02-22 23:05:08' 2026-02-22T23:05:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:10' 2026-02-22T23:05:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:05:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:05:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:05:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:05:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:05:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801506,"replay_state":"idle"} 2026-02-22T23:05:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:08 2026-02-22T23:05:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:05:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:05:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:05:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:10' 2026-02-22T23:05:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:05:15.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:05:15.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:15.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801506,"replay_state":"idle"} 2026-02-22T23:05:15.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:15.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:05:15.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:15.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:05:15.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:15.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:15.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:15.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:08' 2026-02-22T23:05:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:05:08' 2026-02-22T23:05:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:08' 2026-02-22T23:05:15.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:15.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:15.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801506,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:05:08cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:05:10' 2026-02-22T23:05:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:15.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:15.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:08' 2026-02-22T23:05:15.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:05:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:05:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:05:15.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:05:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:05:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:05:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:15.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:05:16.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:05:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:05:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:05:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:05:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:05:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:05:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:05:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:05:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:05:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:05:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:05:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:05:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:05:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:05:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:05:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:05:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:05:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:05:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:05:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:05:17.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:05:17.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:05:17.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:17.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:05:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:05:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:05:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:05:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:05:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:05:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:05:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:05:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:05:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:05:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:05:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:05:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:05:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:05:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:05:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:05:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:05:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:05:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:18 2026-02-22T23:05:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:05:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:05:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:05:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:15' 2026-02-22T23:05:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:05:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:05:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:05:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:18' 2026-02-22T23:05:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:05:18' 2026-02-22T23:05:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:18' 2026-02-22T23:05:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:05:18cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:05:15' 2026-02-22T23:05:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:20.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:18' 2026-02-22T23:05:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:05:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:05:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:05:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:05:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:20 2026-02-22T23:05:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:05:21.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:21.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:21.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:18' 2026-02-22T23:05:21.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:05:21.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:05:21.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:21.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:05:21.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:21.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:21.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:21.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:21.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:20' 2026-02-22T23:05:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:05:20' 2026-02-22T23:05:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:20' 2026-02-22T23:05:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:21.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:21.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:05:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:05:18' 2026-02-22T23:05:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:21.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:20' 2026-02-22T23:05:21.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:21.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:05:21.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:21.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:05:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:05:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:05:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:05:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:05:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:05:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:05:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:05:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:05:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:05:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:05:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:05:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:05:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:05:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:05:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:05:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:05:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:05:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:05:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:05:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:05:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:05:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:05:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:05:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:05:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:05:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 83 2026-02-22T23:05:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:05:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:05:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:05:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:05:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:05:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:05:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:05:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:05:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:05:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:05:29.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:05:30.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=83 2026-02-22T23:05:30.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:05:30.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:05:30.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:05:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:05:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:05:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:05:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=83 2026-02-22T23:05:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 83 = 83 2026-02-22T23:05:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:05:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:05:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:05:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:05:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:05:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:31.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:28 2026-02-22T23:05:31.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:31.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:05:31.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:05:31.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801521,"replay_state":"idle"} 2026-02-22T23:05:31.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:25 2026-02-22T23:05:31.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:05:31.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 83 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.cd07c6b2-ec39-4ae3-b76b-b6ea3fcca672 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:05:31.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:05:31.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:05:31.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:31.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:05:31.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:31.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:05:31.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:31.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:05:31.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:28' 2026-02-22T23:05:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:05:28' 2026-02-22T23:05:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:28' 2026-02-22T23:05:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:31.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:05:28cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801521,"replay_state":"idle"}2026-02-22 23:05:25' 2026-02-22T23:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:31.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:28' 2026-02-22T23:05:31.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:05:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:05:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:05:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:05:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:05:31.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:31.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:31.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:31.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:31.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:31.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:31.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:31.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:32.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:05:32.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:05:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:05:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801521,"replay_state":"idle"} 2026-02-22T23:05:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:25 2026-02-22T23:05:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:05:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:05:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:05:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:28' 2026-02-22T23:05:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:05:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:05:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801521,"replay_state":"idle"} 2026-02-22T23:05:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:05:32.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:32.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:05:32.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:32.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:32.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:32.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:32.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:25' 2026-02-22T23:05:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:05:25' 2026-02-22T23:05:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:25' 2026-02-22T23:05:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:32.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801521,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:05:25cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:05:28' 2026-02-22T23:05:32.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:32.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:25' 2026-02-22T23:05:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:32.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:05:32.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:05:32.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:05:32.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:05:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:05:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:05:32.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:05:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:05:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:05:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:05:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:05:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:05:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:05:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:05:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:05:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:05:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:05:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:05:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:05:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:05:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:05:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:05:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:34.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:34.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:34.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:34.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:05:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:35.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:05:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:33 2026-02-22T23:05:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:05:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:05:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:33' 2026-02-22T23:05:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:05:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:05:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:05:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:35.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:33' 2026-02-22T23:05:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:05:33' 2026-02-22T23:05:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:33' 2026-02-22T23:05:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:05:33cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:05:33' 2026-02-22T23:05:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:33' 2026-02-22T23:05:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:05:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:05:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:05:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:05:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:05:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:35 2026-02-22T23:05:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:05:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:33' 2026-02-22T23:05:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:05:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:05:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:05:36.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:36.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:36.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:35' 2026-02-22T23:05:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:05:35' 2026-02-22T23:05:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:35' 2026-02-22T23:05:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:05:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:05:33' 2026-02-22T23:05:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:35' 2026-02-22T23:05:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:05:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:05:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:05:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:05:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:05:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:05:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:05:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:05:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:05:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:05:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:05:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:05:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:05:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:05:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:05:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:05:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:05:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:05:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:05:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:05:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:05:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:05:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:05:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:39.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:05:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:05:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:05:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:05:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:05:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:05:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:05:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:05:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:05:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:05:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:05:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 87 2026-02-22T23:05:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:05:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:05:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:43.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:05:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:05:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:05:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:05:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:05:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:05:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:05:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:05:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:05:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=87 2026-02-22T23:05:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:05:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:05:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:05:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:05:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:05:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=87 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 87 = 87 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:05:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:40 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801536,"replay_state":"idle"} 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:38 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 87 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.4177f496-484b-4923-b18a-23e3d71a1b3a (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:05:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:05:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:05:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:05:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:40' 2026-02-22T23:05:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:05:40' 2026-02-22T23:05:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:40' 2026-02-22T23:05:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:05:40cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801536,"replay_state":"idle"}2026-02-22 23:05:38' 2026-02-22T23:05:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:44.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:44.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:40' 2026-02-22T23:05:44.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:05:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:05:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:05:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:05:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:05:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:44.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:45.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:05:45.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:45.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:45.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:45.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:45.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:45.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:45.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:45.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:45.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:05:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:05:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801536,"replay_state":"idle"} 2026-02-22T23:05:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:38 2026-02-22T23:05:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:05:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:05:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:05:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:40' 2026-02-22T23:05:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:05:45.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:05:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:45.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801536,"replay_state":"idle"} 2026-02-22T23:05:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:05:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:05:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:38' 2026-02-22T23:05:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:05:38' 2026-02-22T23:05:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:05:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:38' 2026-02-22T23:05:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:45.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:45.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801536,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:05:38cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:05:40' 2026-02-22T23:05:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:45.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:45.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:38' 2026-02-22T23:05:45.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:45.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:05:45.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:05:45.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:05:45.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:05:45.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:45.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:45.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:05:45.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:05:45.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:45.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:45.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:45.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:05:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:05:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:05:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:05:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:05:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:05:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:05:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:05:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:05:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:05:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:05:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:05:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:05:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:05:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:05:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:05:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:05:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:05:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:05:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:05:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:05:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:05:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:05:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:05:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:05:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:05:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:05:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:05:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:05:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:05:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:05:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:05:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:05:49.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:05:49.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:49.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:49.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:05:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:05:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:05:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:05:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:05:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:05:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:49.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:49.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:49.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:49.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:05:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:50.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:50.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:05:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:48 2026-02-22T23:05:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:05:50.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:05:50.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:05:50.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:45' 2026-02-22T23:05:50.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:05:50.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:05:50.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:50.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:05:50.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:50.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:50.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:50.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:48' 2026-02-22T23:05:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:05:48' 2026-02-22T23:05:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:48' 2026-02-22T23:05:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:50.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:05:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:05:45' 2026-02-22T23:05:50.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:50.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:50.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:48' 2026-02-22T23:05:50.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:50.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:05:50.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:50.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:05:50.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:50.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:50.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:05:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:51.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:05:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:05:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:05:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:05:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:50 2026-02-22T23:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:48' 2026-02-22T23:05:51.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:05:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:05:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:05:51.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:05:51.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:05:51.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:51.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:05:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:05:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:05:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:05:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:05:51.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:50' 2026-02-22T23:05:51.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:05:50' 2026-02-22T23:05:51.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:51.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:51.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:05:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:50' 2026-02-22T23:05:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:05:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:05:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:05:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:05:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:05:48' 2026-02-22T23:05:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:05:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:05:51.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:50' 2026-02-22T23:05:51.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:05:51.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:05:51.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:05:51.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:05:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:05:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:05:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:51.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:05:51.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:05:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:05:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:05:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:05:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:05:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:05:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:52.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:05:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:05:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:05:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:05:53.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:05:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:05:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:05:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:05:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:05:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:05:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:05:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:05:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:05:53.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:53.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:53.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:53.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:05:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:05:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:05:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:05:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:05:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:05:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:05:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:05:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:05:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:05:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:05:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:05:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:05:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:05:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:56.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:05:56.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:05:56.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:05:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:05:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:05:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:05:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:05:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:05:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:05:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:05:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:05:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:05:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:05:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:05:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:05:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:05:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:05:56.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:05:56.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:05:56.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:05:56.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:05:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:05:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:56.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:56.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:05:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 89 2026-02-22T23:05:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:05:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:05:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:05:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:05:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:05:58.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:05:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:05:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:05:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:05:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:05:59.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:05:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=89 2026-02-22T23:05:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:05:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:05:59.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:05:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:05:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:05:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:05:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:05:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=89 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 89 = 89 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:05:59.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:06:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:06:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:06:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:06:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:58 2026-02-22T23:06:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:06:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:06:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801551,"replay_state":"idle"} 2026-02-22T23:06:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:55 2026-02-22T23:06:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:06:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 88 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.9e407306-b461-4aea-b396-33144f2a8ee0 (peer_uuids:[]) 2026-02-22T23:06:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 89 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.01ed69d4-2508-4913-b3b8-944dd19bbccc (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:06:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:06:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:06:00.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:00.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:06:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:06:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:06:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:00.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:58' 2026-02-22T23:06:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:05:58' 2026-02-22T23:06:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:58' 2026-02-22T23:06:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:05:58cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801551,"replay_state":"idle"}2026-02-22 23:05:55' 2026-02-22T23:06:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:00.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:58' 2026-02-22T23:06:00.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:06:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:06:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:06:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:06:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:06:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:06:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:06:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:06:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801551,"replay_state":"idle"} 2026-02-22T23:06:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:55 2026-02-22T23:06:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:06:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:06:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:06:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:05:58' 2026-02-22T23:06:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:06:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:06:01.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:01.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801551,"replay_state":"idle"} 2026-02-22T23:06:01.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:01.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:06:01.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:06:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:55' 2026-02-22T23:06:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:05:55' 2026-02-22T23:06:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:01.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:05:55' 2026-02-22T23:06:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:01.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:01.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801551,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:05:55cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:05:58' 2026-02-22T23:06:01.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:01.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:05:55' 2026-02-22T23:06:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:01.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:06:01.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:06:01.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:06:01.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:06:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:06:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:01.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:06:02.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:06:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:06:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:06:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:06:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:06:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:06:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:06:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:06:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:06:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:06:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:06:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:06:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:06:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:06:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:06:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:06:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:06:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:04.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:06:04.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:06:04.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:04.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:06:04.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:04.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:06:04.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:05.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:06:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:05.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:05.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:05.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:06:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:03 2026-02-22T23:06:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:06:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:06:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:03' 2026-02-22T23:06:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:06:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:06:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:06:05.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:05.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:05.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:05.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:05.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:03' 2026-02-22T23:06:05.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:06:03' 2026-02-22T23:06:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:03' 2026-02-22T23:06:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:06:03cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:06:03' 2026-02-22T23:06:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:03' 2026-02-22T23:06:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:06:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:06:05.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:06:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:05.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:06:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:06:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:05 2026-02-22T23:06:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:06:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:03' 2026-02-22T23:06:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:06:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:06:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:06:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:06.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:05' 2026-02-22T23:06:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:06:05' 2026-02-22T23:06:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:05' 2026-02-22T23:06:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:06.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:06.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:06:05cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:06:03' 2026-02-22T23:06:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:05' 2026-02-22T23:06:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:06:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:06:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:06:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:06:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:06:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:06:08.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:06:08.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:08.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:08.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:08.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:06:08.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:06:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:06:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:06:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:06:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:06:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:06:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:06:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:06:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:06:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:06:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:06:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:09.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:06:09.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:06:09.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:09.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:09.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:06:09.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:09.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:09.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:06:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:06:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:06:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:06:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:06:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:06:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:06:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:06:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:09.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:06:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:09.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:09.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:09.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:06:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:06:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:06:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:06:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:06:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:06:13.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 93 2026-02-22T23:06:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:06:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:06:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:06:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:06:13.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:06:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:06:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:06:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:06:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:06:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:06:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:06:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:06:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:06:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:06:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=93 2026-02-22T23:06:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:06:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:06:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:06:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:13.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:13.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:13.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:06:13.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:06:13.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:06:13.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=93 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 93 = 93 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:06:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:06:14.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:06:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:06:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:10 2026-02-22T23:06:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:06:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:06:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801566,"replay_state":"idle"} 2026-02-22T23:06:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:08 2026-02-22T23:06:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:06:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 93 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.1cdf2b31-74ee-4f45-acf5-473f816c9991 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:06:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:06:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:06:14.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:14.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:06:14.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:14.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:06:14.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:14.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:06:14.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:14.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:14.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:14.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:10' 2026-02-22T23:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:06:10' 2026-02-22T23:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:10' 2026-02-22T23:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:14.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:06:10cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801566,"replay_state":"idle"}2026-02-22 23:06:08' 2026-02-22T23:06:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:10' 2026-02-22T23:06:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:06:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:06:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:06:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:06:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:06:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:15.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:06:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:15.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:06:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:06:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801566,"replay_state":"idle"} 2026-02-22T23:06:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:08 2026-02-22T23:06:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:06:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:06:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:06:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:10' 2026-02-22T23:06:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:06:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:06:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801566,"replay_state":"idle"} 2026-02-22T23:06:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:06:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:06:15.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:08' 2026-02-22T23:06:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:06:08' 2026-02-22T23:06:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:08' 2026-02-22T23:06:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801566,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:06:08cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:06:10' 2026-02-22T23:06:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:08' 2026-02-22T23:06:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:06:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:06:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:06:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:06:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:06:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:06:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:15.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:06:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:06:16.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:06:16.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:16.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:16.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:16.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:06:16.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:06:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:06:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:06:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:06:17.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:06:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:06:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:17.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:06:17.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:06:17.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:06:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:06:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:06:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:06:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:06:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:06:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:06:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:06:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:06:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:06:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:06:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:06:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:06:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:06:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:06:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:06:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:06:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:06:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:06:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:06:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:06:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:06:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:06:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:06:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:06:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:19.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:19.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:19.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:19.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:20.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:06:20.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:20.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:20.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:20.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:06:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:18 2026-02-22T23:06:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:06:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:06:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:06:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:15' 2026-02-22T23:06:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:06:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:06:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:06:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:20.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:20.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:20.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:20.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:18' 2026-02-22T23:06:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:06:18' 2026-02-22T23:06:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:18' 2026-02-22T23:06:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:06:18cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:06:15' 2026-02-22T23:06:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:20.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:18' 2026-02-22T23:06:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:06:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:20.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:06:20.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:20.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:06:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:06:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:20 2026-02-22T23:06:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:06:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:18' 2026-02-22T23:06:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:06:21.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:06:21.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:21.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:06:21.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:21.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:21.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:20' 2026-02-22T23:06:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:06:20' 2026-02-22T23:06:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:20' 2026-02-22T23:06:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:06:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:06:18' 2026-02-22T23:06:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:20' 2026-02-22T23:06:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:06:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:06:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:06:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:06:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:06:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:06:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:06:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:22.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:06:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:06:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:06:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:22.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:06:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:06:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:23.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:06:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:06:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:06:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:06:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:06:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:06:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:06:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:06:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:23.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:06:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:06:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:06:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:06:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:24.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:06:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:06:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:06:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:06:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:06:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:06:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:06:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:06:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:06:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:06:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:06:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:06:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:06:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:06:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:06:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:06:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:06:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:06:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:06:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:06:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 95 2026-02-22T23:06:28.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:06:28.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:06:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:06:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:06:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:06:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:06:28.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:06:28.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:06:28.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:06:28.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:28.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:28.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:28.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:28.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:28.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:06:28.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:06:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:06:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=95 2026-02-22T23:06:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:06:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:06:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:06:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:06:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:06:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:06:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:06:29.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:06:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=95 2026-02-22T23:06:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 95 = 95 2026-02-22T23:06:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:06:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:06:30.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:28 2026-02-22T23:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:06:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:06:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801581,"replay_state":"idle"} 2026-02-22T23:06:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:25 2026-02-22T23:06:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:06:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 94 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.44cba6f0-5b98-45b7-aaa6-e2d5ccb7b8f9 (peer_uuids:[]) 2026-02-22T23:06:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 95 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.732ee895-72ba-45fe-b950-abf01ed5eb9f (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:06:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:06:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:06:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:06:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:06:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:06:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:30.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:30.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:28' 2026-02-22T23:06:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:06:28' 2026-02-22T23:06:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:28' 2026-02-22T23:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:30.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:06:28cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801581,"replay_state":"idle"}2026-02-22 23:06:25' 2026-02-22T23:06:30.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:30.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:30.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:28' 2026-02-22T23:06:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:06:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:06:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:06:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:06:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:06:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:06:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:06:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:06:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801581,"replay_state":"idle"} 2026-02-22T23:06:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:25 2026-02-22T23:06:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:06:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:06:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:06:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:28' 2026-02-22T23:06:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:06:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:06:31.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:31.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801581,"replay_state":"idle"} 2026-02-22T23:06:31.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:06:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:06:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:31.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:25' 2026-02-22T23:06:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:06:25' 2026-02-22T23:06:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:31.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:31.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:31.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:25' 2026-02-22T23:06:31.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:31.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:31.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:31.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801581,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:06:25cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:06:28' 2026-02-22T23:06:31.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:31.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:25' 2026-02-22T23:06:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:06:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:06:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:06:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:06:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:06:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:31.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:06:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:06:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:06:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:06:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:06:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:06:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:06:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:06:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:06:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:06:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:06:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:06:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:06:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:33 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:34' 2026-02-22T23:06:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:06:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:06:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:06:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:33' 2026-02-22T23:06:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:06:33' 2026-02-22T23:06:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:33' 2026-02-22T23:06:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:06:33cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:06:34' 2026-02-22T23:06:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:33' 2026-02-22T23:06:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:06:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:06:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:35.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:35.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:35.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:06:35.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:06:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:06:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:35 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:33' 2026-02-22T23:06:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:06:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:06:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:06:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:36.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:36.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:36.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:35' 2026-02-22T23:06:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:06:35' 2026-02-22T23:06:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:35' 2026-02-22T23:06:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:06:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:06:33' 2026-02-22T23:06:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:35' 2026-02-22T23:06:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:06:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:06:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:06:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:06:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:06:38.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:06:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:06:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:06:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:06:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:06:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:06:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:06:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:06:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:06:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:06:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:06:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:06:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:06:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:06:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 99 2026-02-22T23:06:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:06:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:06:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:06:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:06:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:06:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:06:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:06:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:06:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:06:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:06:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:06:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:06:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:06:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:06:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=99 2026-02-22T23:06:43.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:06:43.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:06:43.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:06:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:06:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:06:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:06:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=99 2026-02-22T23:06:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 99 = 99 2026-02-22T23:06:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:06:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:43.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:06:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:06:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:40 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801597,"replay_state":"idle"} 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:38 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:06:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 99 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.148f7e77-4eaa-49a4-a224-4bb39f57d79c (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:06:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:06:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:06:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:06:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:06:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:06:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:40' 2026-02-22T23:06:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:06:40' 2026-02-22T23:06:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:06:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:40' 2026-02-22T23:06:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:06:40cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801597,"replay_state":"idle"}2026-02-22 23:06:38' 2026-02-22T23:06:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:44.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:44.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:40' 2026-02-22T23:06:44.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:44.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:06:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:06:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:06:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:06:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:06:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:45.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:06:45.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:45.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:45.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:45.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:45.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:45.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:45.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:45.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:45.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:06:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:06:45.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801597,"replay_state":"idle"} 2026-02-22T23:06:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:38 2026-02-22T23:06:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:06:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:06:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:06:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:40' 2026-02-22T23:06:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:06:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:06:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801597,"replay_state":"idle"} 2026-02-22T23:06:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:06:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:06:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:38' 2026-02-22T23:06:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:06:38' 2026-02-22T23:06:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:06:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:38' 2026-02-22T23:06:45.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:45.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:45.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:45.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801597,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:06:38cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:06:40' 2026-02-22T23:06:45.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:45.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:45.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:38' 2026-02-22T23:06:45.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:06:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:06:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:06:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:06:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:06:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:06:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:06:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:06:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:06:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:06:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:06:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:06:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:06:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:06:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:06:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:06:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:06:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:06:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:06:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:06:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:06:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:06:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:06:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:06:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:06:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:06:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:06:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:06:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:06:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:06:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:06:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:06:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:49.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:06:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:06:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:06:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:06:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:06:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:06:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:48 2026-02-22T23:06:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:06:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:06:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:06:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:45' 2026-02-22T23:06:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:06:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:06:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:06:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:48' 2026-02-22T23:06:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:06:48' 2026-02-22T23:06:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:50.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:48' 2026-02-22T23:06:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:06:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:06:45' 2026-02-22T23:06:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:48' 2026-02-22T23:06:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:06:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:06:50.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:50.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:50.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:50.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:06:50.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:50.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:50.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:50.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:51.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:06:51.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:51.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:51.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:51.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:06:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:06:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:50 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:06:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:48' 2026-02-22T23:06:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:06:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:06:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:06:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:06:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:06:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:06:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:06:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:06:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:06:51.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:06:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:50' 2026-02-22T23:06:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:06:50' 2026-02-22T23:06:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:06:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:50' 2026-02-22T23:06:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:06:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:06:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:06:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:06:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:06:48' 2026-02-22T23:06:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:06:51.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:06:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:50' 2026-02-22T23:06:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:06:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:06:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:06:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:06:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:06:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:06:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:51.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:06:51.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:06:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:06:53.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:06:53.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:53.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:06:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:53.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:06:53.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:06:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:06:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:06:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:06:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:06:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:06:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:06:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:06:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:53.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:06:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:06:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:06:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:06:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:06:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:06:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:06:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:06:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:06:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:06:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:06:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:06:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:06:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:06:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:06:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:06:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:06:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:06:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:06:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:06:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:06:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:06:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:06:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:06:56.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:06:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:06:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:06:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:06:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:06:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 101 2026-02-22T23:06:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:06:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:06:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:06:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:06:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:06:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:06:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:06:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:06:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:06:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:06:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:06:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:06:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:06:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:06:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:07:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:07:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=101 2026-02-22T23:07:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:07:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:07:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:07:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:07:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:07:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:07:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=101 2026-02-22T23:07:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 101 = 101 2026-02-22T23:07:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:07:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:01.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:07:01.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:01.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:01.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:01.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:01.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:01.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:01.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:01.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:01.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:07:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:58 2026-02-22T23:07:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:07:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:07:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801611,"replay_state":"idle"} 2026-02-22T23:07:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:55 2026-02-22T23:07:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:07:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 101 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.b08355e9-6036-4909-bbfe-ec1b8ba90275 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:07:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:07:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:07:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:07:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:07:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:07:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:58' 2026-02-22T23:07:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:06:58' 2026-02-22T23:07:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:58' 2026-02-22T23:07:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:06:58cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801611,"replay_state":"idle"}2026-02-22 23:06:55' 2026-02-22T23:07:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:58' 2026-02-22T23:07:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:07:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:07:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:07:01.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:01.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:07:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:07:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:07:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801611,"replay_state":"idle"} 2026-02-22T23:07:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:55 2026-02-22T23:07:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:02.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:07:02.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:02.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:02.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:06:58' 2026-02-22T23:07:02.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:07:02.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:07:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801611,"replay_state":"idle"} 2026-02-22T23:07:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:07:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:07:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:55' 2026-02-22T23:07:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:06:55' 2026-02-22T23:07:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:06:55' 2026-02-22T23:07:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801611,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:06:55cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:06:58' 2026-02-22T23:07:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:02.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:02.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:06:55' 2026-02-22T23:07:02.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:07:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:07:02.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:07:02.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:07:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:07:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:07:02.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:07:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:07:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:07:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:07:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:07:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:07:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:07:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:03.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:07:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:07:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:07:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:07:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:07:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:07:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:07:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:07:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:07:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:07:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:03 2026-02-22T23:07:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:07:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:07:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:04' 2026-02-22T23:07:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:07:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:07:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:07:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:05.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:05.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:03' 2026-02-22T23:07:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:07:03' 2026-02-22T23:07:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:03' 2026-02-22T23:07:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:05.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:07:03cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:07:04' 2026-02-22T23:07:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:05.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:03' 2026-02-22T23:07:05.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:07:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:07:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:07:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:07:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:06.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:07:06.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:06.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:06.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:05 2026-02-22T23:07:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:07:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:03' 2026-02-22T23:07:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:07:06.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:07:06.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:06.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:07:06.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:06.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:05' 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:07:05' 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:05' 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:07:05cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:07:03' 2026-02-22T23:07:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:05' 2026-02-22T23:07:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:06.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:07:06.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:06.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:07:06.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:06.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:06.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:06.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:07:06.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:07:08.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:07:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:07:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:07:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:07:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:07:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:07:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:07:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:09.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:07:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:07:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:07:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:07:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:09.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:07:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:07:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:07:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:07:09.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:07:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:09.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:09.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:07:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:07:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:07:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:07:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:07:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:07:09.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:07:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 105 2026-02-22T23:07:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:07:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:07:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:07:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:07:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:07:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:07:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:07:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:07:13.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:07:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:07:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:07:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=105 2026-02-22T23:07:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:07:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:07:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:07:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:07:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:07:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=105 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 105 = 105 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:07:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:14.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:07:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:10 2026-02-22T23:07:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:07:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:07:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801627,"replay_state":"idle"} 2026-02-22T23:07:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:08 2026-02-22T23:07:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:07:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 105 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.6988e1e5-9b14-4536-8278-c147565877f0 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:07:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:07:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:07:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:07:14.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:07:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:07:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:14.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:10' 2026-02-22T23:07:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:07:10' 2026-02-22T23:07:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:10' 2026-02-22T23:07:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:07:10cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801627,"replay_state":"idle"}2026-02-22 23:07:08' 2026-02-22T23:07:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:10' 2026-02-22T23:07:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:07:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:07:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:07:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:15.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801627,"replay_state":"idle"} 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:08 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:10' 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:07:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:07:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801627,"replay_state":"idle"} 2026-02-22T23:07:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:07:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:07:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:08' 2026-02-22T23:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:07:08' 2026-02-22T23:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:08' 2026-02-22T23:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801627,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:07:08cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:07:10' 2026-02-22T23:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:15.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:08' 2026-02-22T23:07:15.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:07:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:07:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:07:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:07:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:07:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:07:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:07:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:07:16.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:07:16.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:07:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:07:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:07:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:07:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:07:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:07:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:07:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:07:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:07:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:07:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:07:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:07:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:07:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:07:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:07:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:07:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:07:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:07:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:07:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:07:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:07:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:07:19.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:07:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:07:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:07:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:07:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:07:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:07:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:07:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:07:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:07:19.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:19.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:19.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:19.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:07:19.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:07:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:07:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:07:19.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:07:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:07:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:07:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:19.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:20.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:07:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:07:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:18 2026-02-22T23:07:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:07:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:15' 2026-02-22T23:07:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:07:20.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:07:20.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:20.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:07:20.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:20.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:20.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:18' 2026-02-22T23:07:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:07:18' 2026-02-22T23:07:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:18' 2026-02-22T23:07:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:20.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:07:18cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:07:15' 2026-02-22T23:07:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:20.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:20.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:18' 2026-02-22T23:07:20.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:07:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:07:20.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:20.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:07:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:07:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:21.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:21.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:21.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:21.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:21.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:21.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:07:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:21.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:20 2026-02-22T23:07:21.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:21.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:07:21.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:21.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:21.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:18' 2026-02-22T23:07:21.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:07:21.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:07:21.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:21.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:07:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:21.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:21.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:21.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:21.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:21.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:20' 2026-02-22T23:07:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:07:20' 2026-02-22T23:07:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:20' 2026-02-22T23:07:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:07:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:07:18' 2026-02-22T23:07:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:21.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:20' 2026-02-22T23:07:21.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:07:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:07:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:07:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:07:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:07:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:07:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:23.973 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:07:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:07:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:07:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:07:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:07:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:07:24.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:07:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:07:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:07:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:07:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:07:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:07:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:07:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:07:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:07:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:07:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:07:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:07:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:07:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:07:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:07:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:07:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:07:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:07:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:07:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:07:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:07:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:07:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:07:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:07:28.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 107 2026-02-22T23:07:28.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:07:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:07:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:07:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:07:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:07:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:07:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:07:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:07:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:07:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:07:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:07:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=107 2026-02-22T23:07:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:07:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:07:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:07:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:07:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:07:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=107 2026-02-22T23:07:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 107 = 107 2026-02-22T23:07:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:07:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:07:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:07:30.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:30.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:30.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:28 2026-02-22T23:07:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:07:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:07:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801641,"replay_state":"idle"} 2026-02-22T23:07:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:25 2026-02-22T23:07:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:07:30.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 106 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.e2033a23-ff67-471a-9f80-124d1635fbab (peer_uuids:[]) 2026-02-22T23:07:30.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 107 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.f69766dc-e2c9-42df-942b-bb718140ff73 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:07:30.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:07:30.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:07:30.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:30.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:07:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:07:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:30.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:07:30.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:30.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:28' 2026-02-22T23:07:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:07:28' 2026-02-22T23:07:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:28' 2026-02-22T23:07:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:07:28cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801641,"replay_state":"idle"}2026-02-22 23:07:25' 2026-02-22T23:07:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:28' 2026-02-22T23:07:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:30.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:07:30.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:07:30.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:07:30.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:07:30.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:31.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:07:31.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:07:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:07:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801641,"replay_state":"idle"} 2026-02-22T23:07:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:25 2026-02-22T23:07:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:07:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:31.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:31.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:28' 2026-02-22T23:07:31.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:07:31.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:07:31.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:31.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801641,"replay_state":"idle"} 2026-02-22T23:07:31.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:07:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:07:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:31.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:31.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:31.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:25' 2026-02-22T23:07:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:07:25' 2026-02-22T23:07:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:25' 2026-02-22T23:07:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801641,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:07:25cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:07:28' 2026-02-22T23:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:25' 2026-02-22T23:07:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:07:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:07:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:07:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:07:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:07:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:07:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:07:32.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:07:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:07:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:07:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:07:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:07:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:07:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:07:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:07:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:07:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:07:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:07:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:07:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:07:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:07:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:07:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:07:33.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:33.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:33.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:33.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:07:34.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:07:34.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:07:34.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:07:34.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:07:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:07:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:07:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:07:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:07:35.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:35.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:35.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:35.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:35.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:35.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:33 2026-02-22T23:07:35.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:35.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:07:35.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:35.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:07:35.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:34' 2026-02-22T23:07:35.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:07:35.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:07:35.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:07:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:35.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:35.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:33' 2026-02-22T23:07:35.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:07:33' 2026-02-22T23:07:35.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:33' 2026-02-22T23:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:07:33cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:07:34' 2026-02-22T23:07:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:33' 2026-02-22T23:07:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:07:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:07:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:07:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:07:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:35 2026-02-22T23:07:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:07:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:33' 2026-02-22T23:07:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:07:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:07:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:07:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:35' 2026-02-22T23:07:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:07:35' 2026-02-22T23:07:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:35' 2026-02-22T23:07:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:07:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:07:33' 2026-02-22T23:07:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:35' 2026-02-22T23:07:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:07:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:07:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:07:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:07:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:07:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:07:38.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:07:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:07:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:07:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:07:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:07:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:39.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:07:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:07:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:07:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:07:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:07:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:07:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:07:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:07:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:07:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:07:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:07:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:07:39.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:39.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:39.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:39.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:07:39.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:07:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:07:39.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:07:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:07:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:07:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:07:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:07:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:07:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:07:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:07:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:07:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:07:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:07:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 111 2026-02-22T23:07:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:07:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:07:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:07:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:07:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:07:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:07:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:07:42.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=111 2026-02-22T23:07:42.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:07:42.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:07:42.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:07:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:07:42.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:07:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=111 2026-02-22T23:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 111 = 111 2026-02-22T23:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:07:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:07:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:40 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801657,"replay_state":"idle"} 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:38 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 110 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.50bf85f5-96c8-4019-af25-b2975f03a83a (peer_uuids:[]) 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 111 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.f3ee1a98-bcbe-4b11-9e7c-8011293d454f (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:07:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:07:43.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:07:43.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:43.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:07:43.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:43.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:07:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:07:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:43.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:43.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:40' 2026-02-22T23:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:07:40' 2026-02-22T23:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:40' 2026-02-22T23:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:43.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:07:40cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801657,"replay_state":"idle"}2026-02-22 23:07:38' 2026-02-22T23:07:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:43.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:40' 2026-02-22T23:07:43.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:07:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:07:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:07:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:07:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:07:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:07:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:07:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:07:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801657,"replay_state":"idle"} 2026-02-22T23:07:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:38 2026-02-22T23:07:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:07:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:40' 2026-02-22T23:07:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:07:44.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:07:44.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:44.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801657,"replay_state":"idle"} 2026-02-22T23:07:44.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:44.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:07:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:07:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:38' 2026-02-22T23:07:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:07:38' 2026-02-22T23:07:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:38' 2026-02-22T23:07:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801657,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:07:38cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:07:40' 2026-02-22T23:07:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:38' 2026-02-22T23:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:07:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:07:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:07:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:07:44.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:44.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:44.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:07:44.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:07:44.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:44.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:44.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:44.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:07:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:07:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:07:46.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:46.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:46.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:46.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:07:46.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:46.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:46.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:46.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:07:46.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:07:46.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:46.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:07:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:07:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:07:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:07:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:47.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:07:47.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:07:47.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:47.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:47.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:47.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:07:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:07:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:07:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:07:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:07:47.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:07:47.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:07:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:07:47.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:07:47.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:07:47.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:07:47.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:07:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:07:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:07:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:48.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:07:48.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:48.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:07:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:07:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:47 2026-02-22T23:07:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:07:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:48.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:45' 2026-02-22T23:07:48.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:07:48.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:07:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image demoted 2026-02-22T23:07:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:48.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:47' 2026-02-22T23:07:48.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:07:47' 2026-02-22T23:07:48.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:48.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:48.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:48.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:48.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:47' 2026-02-22T23:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image demoted43224326mirror.0trial0822026-02-22 23:07:47cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:07:45' 2026-02-22T23:07:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:48.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:47' 2026-02-22T23:07:48.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:48.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:07:48.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:48.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:07:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:07:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:07:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:07:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:45 2026-02-22T23:07:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:49.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:07:49.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:07:49.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:07:49.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:48' 2026-02-22T23:07:49.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:07:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:07:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:07:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:49.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:49.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:49.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:49.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:49.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:45' 2026-02-22T23:07:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:07:45' 2026-02-22T23:07:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:07:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:45' 2026-02-22T23:07:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:07:45cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:07:48' 2026-02-22T23:07:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:45' 2026-02-22T23:07:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:49.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:07:49.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:07:49.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:07:49.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:07:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:07:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:07:49.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:07:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:07:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:07:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:07:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:07:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:07:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:07:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:07:52.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:52.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:52.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:07:52.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:07:52.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:07:52.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:07:52.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:07:52.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:07:52.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:07:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:07:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:07:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:07:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:07:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:07:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:07:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:07:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:07:55.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 113 2026-02-22T23:07:55.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:07:55.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:07:55.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:55.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:55.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:55.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:07:55.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:07:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:07:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:07:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:07:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:07:55.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:55.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:55.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:55.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:55.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:55.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:07:55.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:07:55.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:07:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:07:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=113 2026-02-22T23:07:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:07:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:07:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:07:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:07:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=113 2026-02-22T23:07:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 113 = 113 2026-02-22T23:07:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:07:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:07:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:07:57.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:57.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:57.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:57.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:57.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:57.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:53 2026-02-22T23:07:57.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:57.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:07:57.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:07:57.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801670,"replay_state":"idle"} 2026-02-22T23:07:57.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:50 2026-02-22T23:07:57.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:07:57.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 113 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.5a3bb3c4-3e3b-4fbf-a4de-023e14d9754b (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:07:57.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:07:57.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:07:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:07:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:07:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:57.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:07:57.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:57.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:53' 2026-02-22T23:07:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:07:53' 2026-02-22T23:07:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:07:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:53' 2026-02-22T23:07:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:07:53cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801670,"replay_state":"idle"}2026-02-22 23:07:50' 2026-02-22T23:07:57.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:57.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:57.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:53' 2026-02-22T23:07:57.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:57.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:07:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:07:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:07:58.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:07:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:07:58.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801670,"replay_state":"idle"} 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:50 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:53' 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:07:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:07:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:07:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801670,"replay_state":"idle"} 2026-02-22T23:07:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:07:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:07:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:07:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:07:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:07:58.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:07:58.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:07:58.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:07:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:50' 2026-02-22T23:07:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:07:50' 2026-02-22T23:07:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:07:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:07:50' 2026-02-22T23:07:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:07:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:07:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:07:58.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801670,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:07:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:07:53' 2026-02-22T23:07:58.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:07:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:07:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:07:50' 2026-02-22T23:07:58.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:07:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:07:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:07:58.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:07:58.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:07:58.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:58.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:07:58.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:07:58.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:07:58.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:58.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:58.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:07:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:07:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:07:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:07:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:07:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:07:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:07:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:07:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:07:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:07:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:07:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:07:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:07:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:07:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:07:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:07:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:07:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:07:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:07:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:07:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:07:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:07:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:07:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:07:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:07:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:07:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:07:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:07:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:07:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:07:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:07:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:07:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:07:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:07:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:08:00.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:00.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:00.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:00.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:00.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:00.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:08:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:08:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:08:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:08:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:01.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:08:01.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:01.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:01.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:01.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:01.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:01.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:01.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:58 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:00' 2026-02-22T23:08:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:08:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:08:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:08:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:08:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:08:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:08:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:07:58 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:00' 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:08:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:08:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:08:07.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:07.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:03 2026-02-22T23:08:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:08:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:05' 2026-02-22T23:08:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:08:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:08:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:08:07.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:08:07.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:08:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:08:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:08:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:08:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:08:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:08:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:03' 2026-02-22T23:08:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:08:03' 2026-02-22T23:08:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:03' 2026-02-22T23:08:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:08:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:08:03cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:08:05' 2026-02-22T23:08:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:08:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:08:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:03' 2026-02-22T23:08:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:08:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:08:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:08:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:08:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:07.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:07.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:07.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:07.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:08:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:08:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:05 2026-02-22T23:08:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:08:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:03' 2026-02-22T23:08:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:08:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:08:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:08:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:08:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:08:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:08:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:08:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:08:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:08:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:08:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:05' 2026-02-22T23:08:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:08:05' 2026-02-22T23:08:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:05' 2026-02-22T23:08:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:08:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:08:05cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:08:03' 2026-02-22T23:08:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:08:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:08:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:05' 2026-02-22T23:08:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:08:08.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:08:08.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:08.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:08:08.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:08:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:08:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:08:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:08:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:08:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:08:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:08:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:08:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:08:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:08:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:08:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:08:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:08:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:08:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:08:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:08:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:08:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:08:11.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:11.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:11.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:11.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:08:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:08:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:08:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:11.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:08:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:08:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:08:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:13.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:08:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:08:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:08:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:08:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:08:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:08:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:08:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:08:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:08:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:08:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:08:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:08:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:08:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:08:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:08:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:08:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:08:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:08:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:08:17.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:08:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:08:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:08:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:08:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:08:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:08:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:08:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:08:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:08:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:08:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:08:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:08:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:08:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:08:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:08:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:08:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:17.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:17.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:08:17.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:08:17.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:17.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:08:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:08:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:08:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:08:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:08:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:08:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:08:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:08:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:08:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 117 2026-02-22T23:08:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:08:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:08:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:08:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:08:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:08:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:08:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:08:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:08:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:08:20.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:08:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:08:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:08:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:08:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:08:20.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:08:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=117 2026-02-22T23:08:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:08:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:08:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:08:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:08:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:08:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:08:20.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:08:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:08:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=117 2026-02-22T23:08:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 117 = 117 2026-02-22T23:08:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:08:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:08:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:08:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:08:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:08:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:08:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:08:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:20 2026-02-22T23:08:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:08:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:08:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801688,"replay_state":"idle"} 2026-02-22T23:08:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:13 2026-02-22T23:08:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:08:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 116 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.7c7fbdda-959d-4b0d-af09-f010bc49c015 (peer_uuids:[]) 2026-02-22T23:08:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 117 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.7de3ee01-9412-41d8-8973-51b2ef5840ce (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:08:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:08:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:08:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:08:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:08:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:08:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:08:21.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:08:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:08:21.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:08:21.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:08:21.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:08:21.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:08:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:20' 2026-02-22T23:08:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:08:20' 2026-02-22T23:08:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:08:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:20' 2026-02-22T23:08:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:08:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801688,"replay_state":"idle"}2026-02-22 23:08:13' 2026-02-22T23:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:08:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:08:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:20' 2026-02-22T23:08:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:08:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:08:21.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:08:21.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:08:21.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:08:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:08:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:22.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801688,"replay_state":"idle"} 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:13 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:20' 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:08:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:08:22.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801688,"replay_state":"idle"} 2026-02-22T23:08:22.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:08:22.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:08:22.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:08:22.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:08:22.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:08:22.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:08:22.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:08:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:08:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:13' 2026-02-22T23:08:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:08:13' 2026-02-22T23:08:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:08:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:13' 2026-02-22T23:08:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:22.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:22.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:08:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801688,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:08:13cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:08:20' 2026-02-22T23:08:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:08:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:08:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:13' 2026-02-22T23:08:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:08:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:08:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:08:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:08:22.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:08:22.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:22.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:22.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:08:22.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:08:22.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:22.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:22.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:22.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:08:24.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:24.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:25.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:08:25.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:25.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:25.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:25.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:08:25.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:08:25.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:08:25.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:08:25.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:08:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:08:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:08:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:08:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:08:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:08:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:08:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:08:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:08:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:08:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:08:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:08:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:08:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:08:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:08:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:08:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:08:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:08:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:08:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:08:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:08:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:08:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:08:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:08:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:08:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:25.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:25.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:08:25.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:08:25.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:08:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:08:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:25.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:25.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:25.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:08:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:26.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:08:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:26.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:25 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:25' 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:08:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:08:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image demoted 2026-02-22T23:08:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:08:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:08:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:08:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:08:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:08:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:08:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:25' 2026-02-22T23:08:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:08:25' 2026-02-22T23:08:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:25' 2026-02-22T23:08:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:08:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image demoted43224326mirror.0trial0822026-02-22 23:08:25cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:08:25' 2026-02-22T23:08:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:08:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:08:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:25' 2026-02-22T23:08:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:08:26.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:08:26.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:26.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:08:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:08:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:26.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:08:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:25 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:25' 2026-02-22T23:08:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:08:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:08:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:08:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:08:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:08:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:08:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:08:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:08:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:08:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:08:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:25' 2026-02-22T23:08:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:08:25' 2026-02-22T23:08:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:25' 2026-02-22T23:08:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:08:27.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:08:25cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image demoted2026-02-22 23:08:25' 2026-02-22T23:08:27.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:08:27.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:08:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:25' 2026-02-22T23:08:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:08:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:08:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:08:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:08:28.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:08:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:08:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:08:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:08:29.979 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:08:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:08:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:08:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:08:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:08:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:08:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:08:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:08:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:08:29.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:08:29.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:08:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:08:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:08:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:08:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:08:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:08:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:08:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:08:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:08:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:08:29.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:08:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:08:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:08:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:08:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:08:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:08:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:08:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:08:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:08:29.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:08:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:08:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:08:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:08:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:08:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:08:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:08:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:08:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:08:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:08:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:08:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:08:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:08:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:08:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:08:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:08:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:08:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:08:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:08:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:32.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:08:32.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:08:32.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:08:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:08:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:08:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:08:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:08:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:08:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:08:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:08:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:08:35.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 119 2026-02-22T23:08:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:08:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:08:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:08:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:08:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:08:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:08:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:08:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:08:35.987 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:08:35.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:08:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:08:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:08:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:08:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:08:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:08:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=119 2026-02-22T23:08:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:08:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:08:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:08:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:08:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:08:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:08:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:08:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:08:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=119 2026-02-22T23:08:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 119 = 119 2026-02-22T23:08:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:08:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:08:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:37.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:08:37.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:08:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:08:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:08:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:08:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:33 2026-02-22T23:08:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:08:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:08:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801707,"replay_state":"idle"} 2026-02-22T23:08:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:30 2026-02-22T23:08:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:08:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 119 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.1fdc58eb-1024-41de-8d98-6064175590e0 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:08:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:08:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:08:37.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:08:37.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:08:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:08:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:08:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:08:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:08:37.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:08:37.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:08:37.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:08:37.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:08:37.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:33' 2026-02-22T23:08:37.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:08:33' 2026-02-22T23:08:37.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:37.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:37.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:37.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:08:37.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:33' 2026-02-22T23:08:37.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:37.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:37.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:08:37.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:08:33cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801707,"replay_state":"idle"}2026-02-22 23:08:30' 2026-02-22T23:08:37.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:08:37.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:08:37.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:33' 2026-02-22T23:08:37.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:08:37.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:08:37.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:08:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:08:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:08:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:08:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:08:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:08:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:08:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:08:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:08:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801707,"replay_state":"idle"} 2026-02-22T23:08:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:30 2026-02-22T23:08:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:08:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:08:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:08:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:33' 2026-02-22T23:08:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:08:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:08:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:08:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801707,"replay_state":"idle"} 2026-02-22T23:08:38.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:08:38.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:08:38.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:08:38.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:08:38.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:08:38.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:08:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:08:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:08:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:30' 2026-02-22T23:08:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:08:30' 2026-02-22T23:08:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:08:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:30' 2026-02-22T23:08:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:38.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:38.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:08:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801707,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:08:30cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:08:33' 2026-02-22T23:08:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:08:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:08:38.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:30' 2026-02-22T23:08:38.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:08:38.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:08:38.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:08:38.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:08:38.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:08:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:08:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:08:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:08:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:08:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:08:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:08:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:08:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:08:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:08:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:08:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:08:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:08:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:08:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:08:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:08:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:08:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:08:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:08:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:08:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:08:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:08:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:08:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:08:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:08:40.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:08:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:08:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:08:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:08:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:08:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:08:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:08:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:08:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:08:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:08:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:40.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:08:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:40.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:08:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:41.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:08:41.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:41.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:41.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:08:41.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:08:41.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:41.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:38 2026-02-22T23:08:41.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:41.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:08:41.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:41.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:41.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:40' 2026-02-22T23:08:41.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:08:41.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:08:41.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:41.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:08:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:08:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:38 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:40' 2026-02-22T23:08:43.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:08:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:08:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:08:47.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:08:47.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:47.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:47.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:47.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:47.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:47.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:47.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:47.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:47.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:08:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:47.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:43 2026-02-22T23:08:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:08:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:45' 2026-02-22T23:08:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:08:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:08:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:08:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:08:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:08:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:08:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:08:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:08:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:08:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:08:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:43' 2026-02-22T23:08:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:08:43' 2026-02-22T23:08:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:43' 2026-02-22T23:08:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:08:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:08:43cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:08:45' 2026-02-22T23:08:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:08:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:08:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:43' 2026-02-22T23:08:47.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:08:47.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:08:47.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:47.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:08:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:08:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:08:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:08:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:08:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:08:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:08:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:45 2026-02-22T23:08:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:08:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:08:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:08:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:08:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:43' 2026-02-22T23:08:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:08:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:08:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:08:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:08:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:08:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:08:48.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:48.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:08:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:08:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:08:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:08:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:45' 2026-02-22T23:08:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:08:45' 2026-02-22T23:08:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:08:48.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:45' 2026-02-22T23:08:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:08:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:08:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:08:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:08:45cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:08:43' 2026-02-22T23:08:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:08:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:08:48.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:45' 2026-02-22T23:08:48.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:08:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:08:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:08:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:08:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:08:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:08:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:08:48.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:08:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:08:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:08:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:08:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:08:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:08:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:08:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:08:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:08:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:08:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:08:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:08:51.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:08:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:08:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:08:51.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:08:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:08:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:08:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:08:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:08:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:08:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:08:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:08:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:08:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:08:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:08:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:08:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:08:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:08:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:08:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:08:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:08:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:08:53.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:08:53.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:08:53.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:08:53.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:53.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:08:53.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:08:53.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:08:53.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:08:53.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:08:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:08:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:08:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:08:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:08:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:08:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:08:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:08:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:08:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:08:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:08:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:08:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:08:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:08:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:08:57.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:08:57.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:08:57.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:57.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:08:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:08:57.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:08:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:08:57.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:08:57.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:08:57.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:08:57.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:08:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:08:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:08:57.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:08:57.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:08:57.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:08:57.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:08:57.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:08:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:08:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:08:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:08:57.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:08:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 123 2026-02-22T23:09:00.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:09:00.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:09:00.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:00.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:00.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:09:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:09:00.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:09:00.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:00.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:09:00.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:09:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:09:00.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:00.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:00.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:00.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:00.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:00.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:00.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:09:00.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:09:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:09:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=123 2026-02-22T23:09:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:09:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:09:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:09:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:00.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:09:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:09:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:09:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=123 2026-02-22T23:09:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 123 = 123 2026-02-22T23:09:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:09:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:09:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:09:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:00 2026-02-22T23:09:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:09:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:09:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801728,"replay_state":"idle"} 2026-02-22T23:09:01.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:53 2026-02-22T23:09:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:09:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 122 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.415441ef-7470-4905-ae56-4b0f1b23e881 (peer_uuids:[]) 2026-02-22T23:09:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 123 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.35a6b527-5394-47b7-856c-b91585169e45 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:09:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:09:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:09:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:01.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:09:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:09:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:09:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:00' 2026-02-22T23:09:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:09:00' 2026-02-22T23:09:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:00' 2026-02-22T23:09:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:09:00cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801728,"replay_state":"idle"}2026-02-22 23:08:53' 2026-02-22T23:09:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:00' 2026-02-22T23:09:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:09:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:09:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:09:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:09:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:09:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:01.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:09:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:02.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:02.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:02.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:02.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:09:02.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801728,"replay_state":"idle"} 2026-02-22T23:09:02.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:02.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:08:53 2026-02-22T23:09:02.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:02.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:02.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:02.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:02.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:00' 2026-02-22T23:09:02.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:09:02.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:09:02.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801728,"replay_state":"idle"} 2026-02-22T23:09:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:09:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:09:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:02.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:53' 2026-02-22T23:09:02.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:08:53' 2026-02-22T23:09:02.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:02.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:02.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:02.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:02.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:08:53' 2026-02-22T23:09:02.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:02.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801728,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:08:53cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:09:00' 2026-02-22T23:09:02.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:02.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:08:53' 2026-02-22T23:09:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:02.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:09:02.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:09:02.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:09:02.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:09:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:09:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:09:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:09:04.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:09:04.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:09:04.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:09:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:09:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:09:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:05.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:09:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:09:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:09:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:09:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:09:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:09:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:09:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:09:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:09:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:09:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:09:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:09:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:09:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:09:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:09:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:09:05.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:09:05.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:09:05.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:09:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:09:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:09:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:09:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:09:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:05 2026-02-22T23:09:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:05' 2026-02-22T23:09:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:09:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image demoted 2026-02-22T23:09:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:05' 2026-02-22T23:09:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:09:05' 2026-02-22T23:09:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:06.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:05' 2026-02-22T23:09:06.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:06.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:06.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image demoted43224326mirror.0trial0822026-02-22 23:09:05cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:09:05' 2026-02-22T23:09:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:06.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:06.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:05' 2026-02-22T23:09:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:09:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:09:06.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:06.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:09:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:07.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:09:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:07.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:07.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:05 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:05' 2026-02-22T23:09:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:09:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:09:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:07.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:07.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:07.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:05' 2026-02-22T23:09:07.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:09:05' 2026-02-22T23:09:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:05' 2026-02-22T23:09:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:07.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:09:05cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image demoted2026-02-22 23:09:05' 2026-02-22T23:09:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:05' 2026-02-22T23:09:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:09:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:09:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:09:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:09:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:09:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:09:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:09:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:09:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:09:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:09:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:09.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:09:09.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:09.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:09.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:09.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:09:09.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:09:09.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:09:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:09:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:09:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:09:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:09:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:09:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:09:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:09:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:09:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:09:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:09:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:09:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:09:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:09:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:09:09.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:09:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:09:09.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:09:09.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:09:09.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:09.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:09.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:09.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:09:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:09:10.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:09:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:09:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:09:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:09:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:09:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:09:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:09:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:09:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:09:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:09:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:09:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:09:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:09:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:09:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:09:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:09:12.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:09:12.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:09:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:09:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:09:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:09:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:09:12.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:09:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:09:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:09:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:09:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:09:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:09:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:09:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:09:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:09:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:09:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:09:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:09:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:09:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:09:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 125 2026-02-22T23:09:15.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:09:15.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:09:15.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:15.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:09:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:09:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:09:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:09:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:09:15.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:09:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:09:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:09:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:09:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=125 2026-02-22T23:09:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:09:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:09:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:09:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:09:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:09:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:09:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=125 2026-02-22T23:09:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 125 = 125 2026-02-22T23:09:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:09:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:09:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:17.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:17.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:17.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:17.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:17.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:13 2026-02-22T23:09:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801752,"remote_snapshot_timestamp":1771801752,"replay_state":"idle"} 2026-02-22T23:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:15 2026-02-22T23:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 125 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.4f02cdf2-fbac-46f7-bece-3927ddfebb9f (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:09:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:09:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:09:17.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:17.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:09:17.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:17.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:09:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:13' 2026-02-22T23:09:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:09:13' 2026-02-22T23:09:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:13' 2026-02-22T23:09:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:17.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:09:13cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801752,"remote_snapshot_timestamp":1771801752,"replay_state":"idle"}2026-02-22 23:09:15' 2026-02-22T23:09:17.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:13' 2026-02-22T23:09:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:09:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:09:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:09:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:09:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:09:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:09:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:18.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:18.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:09:18.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:18.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:18.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:09:18.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801752,"remote_snapshot_timestamp":1771801752,"replay_state":"idle"} 2026-02-22T23:09:18.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:18.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:15 2026-02-22T23:09:18.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:09:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:13' 2026-02-22T23:09:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:09:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:09:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801752,"remote_snapshot_timestamp":1771801752,"replay_state":"idle"} 2026-02-22T23:09:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:09:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:18.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:09:18.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:15' 2026-02-22T23:09:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:09:15' 2026-02-22T23:09:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:18.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:15' 2026-02-22T23:09:18.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:18.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:18.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801752,"remote_snapshot_timestamp":1771801752,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:09:15cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:09:13' 2026-02-22T23:09:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:18.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:15' 2026-02-22T23:09:18.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:09:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:09:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:09:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:09:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:09:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:18.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:18.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:18.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:09:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:09:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:09:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:09:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:09:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:09:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:09:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:09:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:09:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:09:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:09:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:09:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:09:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:09:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:09:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:09:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:09:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:09:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:09:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:09:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:09:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:09:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:09:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:09:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:09:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:09:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:09:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:09:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:09:20.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:20.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:20.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:09:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:09:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:09:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:09:20.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:09:20.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:09:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:09:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:09:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:09:22.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:09:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:09:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:09:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:09:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:09:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:09:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:09:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:09:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:09:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:09:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:09:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:09:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:09:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:09:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:09:22.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:22.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:22.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:22.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:09:22.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:09:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:09:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:09:22.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:09:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:09:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:09:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:22.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:22.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:09:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:18 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:20' 2026-02-22T23:09:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:09:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:09:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:09:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:23 2026-02-22T23:09:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:09:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:20' 2026-02-22T23:09:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:09:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:09:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:23' 2026-02-22T23:09:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:09:23' 2026-02-22T23:09:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:23' 2026-02-22T23:09:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:25.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:09:23cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:09:20' 2026-02-22T23:09:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:23' 2026-02-22T23:09:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:25.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:09:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:09:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:09:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:09:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:09:26.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:25 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:23' 2026-02-22T23:09:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:09:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:09:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:26.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:25' 2026-02-22T23:09:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:09:25' 2026-02-22T23:09:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:25' 2026-02-22T23:09:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:09:25cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:09:23' 2026-02-22T23:09:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:25' 2026-02-22T23:09:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:09:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:09:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:09:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:09:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:09:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:09:28.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:09:28.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:28.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:28.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:28.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:09:28.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:28.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:28.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:28.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:09:28.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:09:28.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:09:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:09:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:09:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:09:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:09:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:09:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:09:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:09:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:09:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:09:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:09:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:09:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:09:29.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:09:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:09:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:09:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:09:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:09:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:09:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:09:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:09:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:09:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:09:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:09:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:09:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:09:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:09:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:09:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:09:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:09:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 129 2026-02-22T23:09:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:09:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:09:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:09:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:09:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:09:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:09:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:09:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:09:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:09:32.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:09:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:09:32.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=129 2026-02-22T23:09:32.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:09:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:09:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:09:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:09:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:09:32.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:09:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=129 2026-02-22T23:09:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 129 = 129 2026-02-22T23:09:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:09:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:09:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:09:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:30 2026-02-22T23:09:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:09:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:09:33.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801766,"replay_state":"idle"} 2026-02-22T23:09:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:28 2026-02-22T23:09:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:09:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 128 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.3905bb8b-adcb-4c2b-8b28-ee998552d4e8 (peer_uuids:[]) 2026-02-22T23:09:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 129 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.7892291c-3aee-4609-a29b-27eea9e4d404 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:09:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:09:33.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:09:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:09:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:09:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:09:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:33.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:33.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:33.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:30' 2026-02-22T23:09:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:09:30' 2026-02-22T23:09:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:30' 2026-02-22T23:09:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:33.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:09:30cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801766,"replay_state":"idle"}2026-02-22 23:09:28' 2026-02-22T23:09:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:30' 2026-02-22T23:09:33.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:09:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:09:33.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:09:33.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:09:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:09:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:09:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:34.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801766,"replay_state":"idle"} 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:28 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:30' 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:09:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:34.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801766,"replay_state":"idle"} 2026-02-22T23:09:34.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:34.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:09:34.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:09:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:28' 2026-02-22T23:09:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:09:28' 2026-02-22T23:09:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:28' 2026-02-22T23:09:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:34.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801766,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:09:28cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:09:30' 2026-02-22T23:09:34.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:34.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:28' 2026-02-22T23:09:34.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:34.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:09:34.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:09:34.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:09:34.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:09:34.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:34.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:34.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:09:34.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:09:34.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:34.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:34.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:34.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:09:36.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:09:36.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:09:36.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:36.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:36.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:36.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:09:36.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:36.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:36.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:36.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:09:36.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:09:36.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:36.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:09:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:09:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:09:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:09:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:09:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:09:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:09:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:09:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:09:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:09:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:09:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:09:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:09:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:09:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:09:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:09:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:09:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:09:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:09:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:09:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:09:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:09:37.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:09:37.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:09:37.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:09:37.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:09:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:09:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:09:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:09:37.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:09:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:09:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:09:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:09:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:37 2026-02-22T23:09:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:35' 2026-02-22T23:09:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:09:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image demoted 2026-02-22T23:09:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:37' 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:09:37' 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:37' 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:38.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:38.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image demoted43224326mirror.0trial0822026-02-22 23:09:37cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:09:35' 2026-02-22T23:09:38.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:38.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:38.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:37' 2026-02-22T23:09:38.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:38.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:09:38.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:38.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:09:38.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:09:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:38.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:09:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:39.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:35 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:38' 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:09:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:09:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:39.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:39.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:39.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:39.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:35' 2026-02-22T23:09:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:09:35' 2026-02-22T23:09:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:39.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:35' 2026-02-22T23:09:39.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:39.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:39.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:39.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:09:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:09:38' 2026-02-22T23:09:39.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:35' 2026-02-22T23:09:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:39.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:09:39.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:39.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:09:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:09:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:09:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:09:41.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:09:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:09:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:09:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:09:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:09:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:09:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:09:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:09:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:09:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:09:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:09:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:09:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:09:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:09:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:09:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:09:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:09:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:09:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:09:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:09:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:09:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:09:42.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:09:42.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:09:42.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:09:42.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:09:42.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:09:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 131 2026-02-22T23:09:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:09:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:09:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:09:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:09:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:09:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:09:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:09:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:09:45.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:45.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:45.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:45.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:45.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:45.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:09:45.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:09:45.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:09:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:09:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=131 2026-02-22T23:09:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:09:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:09:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:09:46.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:46.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:09:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:09:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:09:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:09:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=131 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 131 = 131 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:09:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:47.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:47.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:47.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:47.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:47.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:47.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:43 2026-02-22T23:09:47.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:47.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:47.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:09:47.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801782,"remote_snapshot_timestamp":1771801782,"replay_state":"idle"} 2026-02-22T23:09:47.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:45 2026-02-22T23:09:47.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:09:47.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 131 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.7045d46c-836e-4c79-ba8e-f31e071adebb (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:09:47.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:09:47.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:09:47.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:47.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:09:47.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:47.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:09:47.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:47.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:09:47.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:43' 2026-02-22T23:09:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:09:43' 2026-02-22T23:09:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:09:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:43' 2026-02-22T23:09:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:47.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:09:43cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801782,"remote_snapshot_timestamp":1771801782,"replay_state":"idle"}2026-02-22 23:09:45' 2026-02-22T23:09:47.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:47.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:47.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:43' 2026-02-22T23:09:47.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:09:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:09:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:09:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:09:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:09:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:47.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:47.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:47.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:47.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:09:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:09:48.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:48.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:48.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:09:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801782,"remote_snapshot_timestamp":1771801782,"replay_state":"idle"} 2026-02-22T23:09:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:45 2026-02-22T23:09:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:09:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:43' 2026-02-22T23:09:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:09:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801782,"remote_snapshot_timestamp":1771801782,"replay_state":"idle"} 2026-02-22T23:09:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:09:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:09:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:45' 2026-02-22T23:09:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:09:45' 2026-02-22T23:09:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:09:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:45' 2026-02-22T23:09:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:48.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:48.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801782,"remote_snapshot_timestamp":1771801782,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:09:45cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:09:43' 2026-02-22T23:09:48.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:48.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:48.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:45' 2026-02-22T23:09:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:09:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:09:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:09:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:09:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:09:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:09:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:09:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:09:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:09:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:09:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:09:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:09:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:09:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:09:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:09:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:09:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:09:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:09:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:09:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:09:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:09:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:09:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:09:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:09:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:09:50.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:50.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:50.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:09:50.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:09:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:09:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:09:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:09:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:09:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:09:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:09:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:51.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:09:51.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:51.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:48 2026-02-22T23:09:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:50' 2026-02-22T23:09:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:09:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:09:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:09:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:53.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:09:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:09:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:48 2026-02-22T23:09:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:50' 2026-02-22T23:09:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:09:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:09:57.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:09:57.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:57.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:57.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:57.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:09:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:57.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:53 2026-02-22T23:09:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:09:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:55' 2026-02-22T23:09:57.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:57.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:09:57.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:57.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:09:57.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:57.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:57.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:57.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:57.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:57.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:57.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:53' 2026-02-22T23:09:57.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:09:53' 2026-02-22T23:09:57.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:09:57.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:57.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:57.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:57.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:53' 2026-02-22T23:09:57.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:57.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:57.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:09:53cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:09:55' 2026-02-22T23:09:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:57.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:53' 2026-02-22T23:09:57.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:57.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:09:57.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:57.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:09:57.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:09:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:09:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:09:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:09:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:09:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:09:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:09:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:09:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:09:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:09:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:55 2026-02-22T23:09:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:09:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:09:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:09:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:09:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:09:53' 2026-02-22T23:09:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:09:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:09:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:09:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:09:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:09:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:09:58.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:58.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:09:58.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:09:58.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:09:58.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:09:58.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:55' 2026-02-22T23:09:58.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:09:55' 2026-02-22T23:09:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:09:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:09:55' 2026-02-22T23:09:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:09:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:09:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:09:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:09:55cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:09:53' 2026-02-22T23:09:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:09:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:09:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:09:55' 2026-02-22T23:09:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:09:58.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:09:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:09:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:09:58.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:09:58.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:58.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:09:58.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:09:58.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:09:58.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:09:58.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:09:58.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:09:58.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:10:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:10:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:10:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:10:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:10:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:10:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:10:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:01.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:10:01.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:10:01.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:01.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:01.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:01.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:01.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:01.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:10:01.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:10:01.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:01.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:10:01.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:01.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:10:03.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:10:03.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:10:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:10:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:10:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:03.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:03.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:10:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:03.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:10:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:10:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:10:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:10:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:10:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:10:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:10:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:10:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:10:07.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:07.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:07.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:10:07.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:10:07.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:07.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:07.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:07.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:10:07.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:07.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:07.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:07.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:10:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:10:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:10:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:10:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:10:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:10:10.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 135 2026-02-22T23:10:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:10:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:10:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:10:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:10:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:10:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:10.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:10:10.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:10:10.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:10:10.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:10.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:10.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:10:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:10:10.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:10:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=135 2026-02-22T23:10:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:10:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:10:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:10:10.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:10:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=135 2026-02-22T23:10:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 135 = 135 2026-02-22T23:10:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:10:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:10.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:10.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:10.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:10:10.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:10.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:11.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:10:11.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:11.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:11.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:11.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:10:11.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:11.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:11.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:11.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:11.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:10 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801799,"replay_state":"idle"} 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:03 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 134 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.7a76dba9-8125-41a7-9c56-dd44b3e06ed0 (peer_uuids:[]) 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 135 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.bd701a8a-d3f1-43cb-bd98-0e3485d40c87 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:10:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:10:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:10:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:10:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:10:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:10' 2026-02-22T23:10:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:10:10' 2026-02-22T23:10:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:10:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:10' 2026-02-22T23:10:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:11.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:10:10cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801799,"replay_state":"idle"}2026-02-22 23:10:03' 2026-02-22T23:10:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:11.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:10' 2026-02-22T23:10:11.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:11.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:10:11.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:10:11.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:10:11.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:10:11.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:11.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:11.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:10:11.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:11.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:11.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:11.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:10:11.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:11.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:11.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:11.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:10:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:10:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:10:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:10:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801799,"replay_state":"idle"} 2026-02-22T23:10:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:03 2026-02-22T23:10:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:10:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:10:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:10:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:10' 2026-02-22T23:10:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:10:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:10:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801799,"replay_state":"idle"} 2026-02-22T23:10:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:10:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:10:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:12.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:03' 2026-02-22T23:10:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:10:03' 2026-02-22T23:10:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:10:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:03' 2026-02-22T23:10:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801799,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:10:03cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:10:10' 2026-02-22T23:10:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:12.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:03' 2026-02-22T23:10:12.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:12.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:10:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:10:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:10:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:10:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:10:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:10:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:10:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:10:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:10:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:10:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:10:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:10:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:15.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:15.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:15.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:15.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:10:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:10:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:10:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:16.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:10:16.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:16.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:16.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:16.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:10:16.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:16.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:15 2026-02-22T23:10:16.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:16.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:10:16.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:16.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:10:16.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:15' 2026-02-22T23:10:16.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:10:16.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:10:16.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:16.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image demoted 2026-02-22T23:10:16.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:15' 2026-02-22T23:10:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:10:15' 2026-02-22T23:10:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:15' 2026-02-22T23:10:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image demoted43224326mirror.0trial0822026-02-22 23:10:15cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:10:15' 2026-02-22T23:10:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:15' 2026-02-22T23:10:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:10:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:10:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:10:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:10:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:17.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:10:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:10:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:15 2026-02-22T23:10:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:10:17.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:17.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:10:17.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:15' 2026-02-22T23:10:17.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:10:17.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:10:17.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:17.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:10:17.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:17.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:17.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:17.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:17.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:17.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:17.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:17.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:15' 2026-02-22T23:10:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:10:15' 2026-02-22T23:10:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:15' 2026-02-22T23:10:17.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:17.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:17.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:10:15cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image demoted2026-02-22 23:10:15' 2026-02-22T23:10:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:15' 2026-02-22T23:10:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:17.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:10:17.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:17.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:10:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:10:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:10:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:10:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:10:18.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:10:18.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:18.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:18.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:10:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:10:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:10:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:10:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:10:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:10:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:10:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:10:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:10:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:10:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:10:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:10:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:10:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:10:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:10:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:10:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:10:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:10:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:10:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:22.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:10:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:10:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:10:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:10:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:10:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:10:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:10:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:10:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:10:25.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 137 2026-02-22T23:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:10:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:10:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:10:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:10:25.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:10:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:25.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:10:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:10:26.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:10:26.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=137 2026-02-22T23:10:26.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:10:26.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:10:26.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:10:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:10:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:10:26.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=137 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 137 = 137 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:10:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:10:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:10:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:10:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:10:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:23 2026-02-22T23:10:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:10:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:10:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801817,"replay_state":"idle"} 2026-02-22T23:10:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:20 2026-02-22T23:10:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:10:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 137 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.a88b12b9-7ac5-435a-917e-78b08e95d173 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:10:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:10:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:10:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:10:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:10:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:10:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:23' 2026-02-22T23:10:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:10:23' 2026-02-22T23:10:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:10:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:23' 2026-02-22T23:10:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:10:23cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801817,"replay_state":"idle"}2026-02-22 23:10:20' 2026-02-22T23:10:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:27.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:23' 2026-02-22T23:10:27.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:10:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:10:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:10:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:10:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:10:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:10:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801817,"replay_state":"idle"} 2026-02-22T23:10:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:20 2026-02-22T23:10:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:10:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:10:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:10:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:23' 2026-02-22T23:10:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:10:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:10:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801817,"replay_state":"idle"} 2026-02-22T23:10:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:10:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:10:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:20' 2026-02-22T23:10:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:10:20' 2026-02-22T23:10:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:10:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:20' 2026-02-22T23:10:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801817,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:10:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:10:23' 2026-02-22T23:10:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:28.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:20' 2026-02-22T23:10:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:10:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:10:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:10:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:10:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:10:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:10:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:30.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:10:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:10:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:10:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:10:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:10:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:10:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:10:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:10:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:10:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:10:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:10:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:10:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:10:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:10:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:10:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:10:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:10:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:10:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:10:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:10:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:10:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:10:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:10:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:10:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:10:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:10:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:10:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:10:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:10:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:10:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:10:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:10:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:10:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:10:32.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:32.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:32.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:32.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:10:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:10:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:10:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:10:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:10:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:33.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:10:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:10:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:10:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:28 2026-02-22T23:10:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:10:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:10:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:31' 2026-02-22T23:10:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:10:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:10:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:10:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:10:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:33 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:31' 2026-02-22T23:10:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:10:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:10:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:10:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:35.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:35.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:33' 2026-02-22T23:10:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:10:33' 2026-02-22T23:10:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:33' 2026-02-22T23:10:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:10:33cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:10:31' 2026-02-22T23:10:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:33' 2026-02-22T23:10:35.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:35.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:10:35.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:35.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:10:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:10:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:35 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:33' 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:10:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:10:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:10:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:35' 2026-02-22T23:10:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:10:35' 2026-02-22T23:10:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:35' 2026-02-22T23:10:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:36.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:10:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:10:33' 2026-02-22T23:10:36.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:36.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:35' 2026-02-22T23:10:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:10:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:10:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:10:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:10:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:10:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:10:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:10:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:10:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:10:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:10:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:10:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:10:39.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:39.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:10:39.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:10:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:10:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:10:39.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:10:39.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:10:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 141 2026-02-22T23:10:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:10:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:10:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:10:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:10:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:10:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:10:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:10:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:10:43.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:43.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:43.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:43.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:43.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:43.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:43.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:10:43.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:10:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:10:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=141 2026-02-22T23:10:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:10:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:10:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:10:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:43.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:10:43.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:10:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=141 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 141 = 141 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:10:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:10:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:10:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:44.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:40 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801837,"replay_state":"idle"} 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:38 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:10:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 141 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.840e4c04-87a8-4096-9fe3-d0b52b0998a9 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:10:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:10:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:10:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:44.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:10:44.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:44.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:10:44.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:44.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:10:44.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:44.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:44.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:44.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:44.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:40' 2026-02-22T23:10:44.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:10:40' 2026-02-22T23:10:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:10:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:40' 2026-02-22T23:10:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:44.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:10:40cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801837,"replay_state":"idle"}2026-02-22 23:10:38' 2026-02-22T23:10:44.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:44.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:40' 2026-02-22T23:10:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:44.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:10:44.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:10:44.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:10:44.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:10:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:10:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:10:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:10:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:10:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:10:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:10:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801837,"replay_state":"idle"} 2026-02-22T23:10:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:38 2026-02-22T23:10:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:10:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:10:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:10:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:40' 2026-02-22T23:10:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:10:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:10:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801837,"replay_state":"idle"} 2026-02-22T23:10:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:10:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:10:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:45.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:38' 2026-02-22T23:10:45.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:10:38' 2026-02-22T23:10:45.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:45.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:10:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:38' 2026-02-22T23:10:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801837,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:10:38cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:10:40' 2026-02-22T23:10:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:38' 2026-02-22T23:10:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:10:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:10:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:10:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:10:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:10:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test 2026-02-22T23:10:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T23:10:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:10:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:10:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:47.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:10:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:10:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:10:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = started 2026-02-22T23:10:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:10:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:10:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:10:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:10:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:10:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:10:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:10:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:10:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:10:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:10:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:10:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:10:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:10:49.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:10:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:49.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:49.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:10:49.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:10:49.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:10:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:10:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:48 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:45' 2026-02-22T23:10:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:10:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:10:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:10:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:50.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:50.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:50.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:48' 2026-02-22T23:10:50.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:10:48' 2026-02-22T23:10:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:48' 2026-02-22T23:10:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:50.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:10:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:10:45' 2026-02-22T23:10:50.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:50.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:48' 2026-02-22T23:10:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:10:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:10:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:51.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:10:51.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:51.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:51.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:51.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:51.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:51.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:51.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:51.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:51.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:10:51.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:10:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:10:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:10:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:10:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:50 2026-02-22T23:10:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:10:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:10:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:10:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:10:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:48' 2026-02-22T23:10:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:10:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:10:51.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:10:51.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:10:51.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:10:51.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:10:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:10:51.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:10:51.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:10:51.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:10:51.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:50' 2026-02-22T23:10:51.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:10:50' 2026-02-22T23:10:51.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:51.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:51.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:51.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:10:51.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:50' 2026-02-22T23:10:51.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:10:51.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:10:51.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:10:51.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:10:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:10:48' 2026-02-22T23:10:51.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:10:51.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:10:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:50' 2026-02-22T23:10:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:10:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:10:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:10:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:10:51.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:10:51.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:51.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:51.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test 2026-02-22T23:10:51.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:51.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:51.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:51.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:10:51.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T23:10:52.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:10:52.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T23:10:52.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T23:10:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:10:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:10:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:10:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:10:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:10:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:10:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:10:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:10:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:10:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:10:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:10:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:10:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:10:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:10:54.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:54.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:54.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:54.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:10:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:10:54.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:10:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:10:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T23:10:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:10:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:10:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:10:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:10:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:10:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:10:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:10:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:10:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:10:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:10:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:10:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:10:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:10:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:10:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:10:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:10:56.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:10:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:10:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:10:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:10:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:10:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:10:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:10:56.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:10:56.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:10:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:10:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:10:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:10:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:10:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:10:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T23:10:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:10:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:10:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:10:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-22T23:10:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-22T23:10:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 143 2026-02-22T23:10:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror test 2026-02-22T23:10:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster2 2026-02-22T23:10:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:10:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:10:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:10:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:10:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:10:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:10:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:10:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:10:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:10:59.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:10:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=143 2026-02-22T23:10:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:10:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:10:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:10:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:10:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:10:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:10:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:10:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:10:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=143 2026-02-22T23:10:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 143 = 143 2026-02-22T23:10:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:10:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T23:10:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:10:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:10:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:10:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:10:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:10:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:10:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:10:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T23:11:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:11:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:11:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:11:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:11:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:11:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:58 2026-02-22T23:11:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:11:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:11:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801852,"replay_state":"idle"} 2026-02-22T23:11:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:55 2026-02-22T23:11:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:11:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 142 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.9dba53fb-a500-4796-91d3-688e437da54e (peer_uuids:[]) 2026-02-22T23:11:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 143 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.763f4579-ee48-416c-b18c-3b39a70044f5 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:11:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:11:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:11:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:11:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:11:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:11:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:11:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:58' 2026-02-22T23:11:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 23:10:58' 2026-02-22T23:11:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:58' 2026-02-22T23:11:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:00.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary43224326mirror.0trial0822026-02-22 23:10:58cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801852,"replay_state":"idle"}2026-02-22 23:10:55' 2026-02-22T23:11:00.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:00.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:11:00.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:58' 2026-02-22T23:11:00.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:11:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:11:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:11:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T23:11:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:11:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:11:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:11:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:11:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801852,"replay_state":"idle"} 2026-02-22T23:11:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:11:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:55 2026-02-22T23:11:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:11:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:11:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:10:58' 2026-02-22T23:11:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:11:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:11:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801852,"replay_state":"idle"} 2026-02-22T23:11:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:11:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:11:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:11:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:55' 2026-02-22T23:11:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 23:10:55' 2026-02-22T23:11:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:10:55' 2026-02-22T23:11:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:01.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:01.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:01.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801852,"replay_state":"idle"}44464451mirror.0trial0822026-02-22 23:10:55cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:10:58' 2026-02-22T23:11:01.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:01.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:11:01.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:10:55' 2026-02-22T23:11:01.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:11:01.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:11:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:11:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:11:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:11:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror test 2026-02-22T23:11:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T23:11:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:11:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T23:11:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T23:11:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:11:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:11:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:03.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:11:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:11:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:11:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:11:03.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:11:03.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:03.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:04.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:04.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:04.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:11:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:11:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:11:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:11:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:11:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:11:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:11:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T23:11:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:11:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T23:11:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:11:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:11:05.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:11:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:11:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:11:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:11:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:11:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:03 2026-02-22T23:11:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:11:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:11:05.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:11:05.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:04' 2026-02-22T23:11:05.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:11:05.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:11:05.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:05.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:11:05.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:05.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:11:05.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:11:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:11:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:03' 2026-02-22T23:11:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 23:11:03' 2026-02-22T23:11:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:11:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:11:03' 2026-02-22T23:11:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary43224326mirror.0trial0822026-02-22 23:11:03cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image demoted2026-02-22 23:11:04' 2026-02-22T23:11:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:05.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:11:05.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:03' 2026-02-22T23:11:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:11:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:11:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:11:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:11:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T23:11:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:11:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:11:06.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:11:06.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:05 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:03' 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:11:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:11:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:11:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:11:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:11:06.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:05' 2026-02-22T23:11:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 23:11:05' 2026-02-22T23:11:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:11:05' 2026-02-22T23:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:06.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+unknownremote image is not primary44464451mirror.0trial0822026-02-22 23:11:05cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:11:03' 2026-02-22T23:11:06.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:06.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:11:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:05' 2026-02-22T23:11:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:11:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:11:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:11:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:11:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:11:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster2 mirror test 2026-02-22T23:11:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:11:06.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T23:11:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:11:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:11:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:11:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:11:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:11:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:11:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:11:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:11:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:11:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:11:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:11:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:11:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:11:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:11:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:11:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:11:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:11:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:09.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:11:09.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:09.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:09.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:09.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:11:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:11:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:11:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:11:09.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:11:09.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:11:09.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:11:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:11:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T23:11:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:11:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:09.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:09.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:11:09.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:11:09.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-22T23:11:09.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:09.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:09.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:09.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-22T23:11:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 147 2026-02-22T23:11:12.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2026-02-22T23:11:12.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:11:12.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:12.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:12.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:12.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.status 2026-02-22T23:11:12.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.status 2026-02-22T23:11:12.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:11:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.status 2026-02-22T23:11:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.status 2026-02-22T23:11:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:11:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:11:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.status 2026-02-22T23:11:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-22T23:11:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:12.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:11:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=147 2026-02-22T23:11:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:11:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:11:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:11:12.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:11:12.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:12.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:12.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:12.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.status 2026-02-22T23:11:12.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-22T23:11:12.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:11:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=147 2026-02-22T23:11:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 147 = 147 2026-02-22T23:11:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T23:11:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T23:11:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:13.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:10 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801867,"replay_state":"idle"} 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:08 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 146 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.f9b4ec41-5192-4f7d-b828-9643580320ea (peer_uuids:[]) 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 147 .mirror.primary.d73b653c-f1e8-487c-91fe-8129b425b941.ccb150be-6710-4320-90be-0ec23acf58d6 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:11:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:11:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:11:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:11:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:11:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:11:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:10' 2026-02-22T23:11:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 23:11:10' 2026-02-22T23:11:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:11:10' 2026-02-22T23:11:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:13.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:11:10cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801867,"replay_state":"idle"}2026-02-22 23:11:08' 2026-02-22T23:11:13.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:13.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:11:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:10' 2026-02-22T23:11:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:11:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:11:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:11:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:11:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:11:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:11:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:11:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d73b653c-f1e8-487c-91fe-8129b425b941 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801867,"replay_state":"idle"} 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:08 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:10' 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:11:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801867,"replay_state":"idle"} 2026-02-22T23:11:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:11:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:11:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:11:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:08' 2026-02-22T23:11:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:11:08' 2026-02-22T23:11:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:11:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:11:08' 2026-02-22T23:11:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:14.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801867,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:11:08cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:11:10' 2026-02-22T23:11:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:14.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:11:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:08' 2026-02-22T23:11:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:11:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:11:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:11:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:11:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:11:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:11:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_mirror_snaps cluster1 mirror test 2026-02-22T23:11:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:11:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:11:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:11:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 snap ls mirror/test --all 2026-02-22T23:11:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c -F ' mirror (' 2026-02-22T23:11:14.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 3 -le 3 2026-02-22T23:11:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_mirror_snaps cluster2 mirror test 2026-02-22T23:11:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:11:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:11:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:11:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 snap ls mirror/test --all 2026-02-22T23:11:14.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c -F ' mirror (' 2026-02-22T23:11:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 2 -le 3 2026-02-22T23:11:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: force promote' 2026-02-22T23:11:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:11:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:11:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:11:14 TEST: force promote 2026-02-22T23:11:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:11:14 TEST: force promote 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ force_promote_image=test_force_promote 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test_force_promote snapshot 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:11:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test_force_promote 2026-02-22T23:11:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:11:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:14.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test_force_promote 2026-02-22T23:11:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:11:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:11:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test_force_promote snapshot 2026-02-22T23:11:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:14.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:14.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:11:14.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_force_promote snapshot 2026-02-22T23:11:15.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:11:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test_force_promote 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_force_promote': 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 14fd71c7c151 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.14fd71c7c151 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:11:14 2026 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:11:14 2026 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:11:14 2026 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:11:15.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:11:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: c3ac37a3-ca45-4496-b917-0cf3f14c618d 2026-02-22T23:11:15.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:11:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_force_promote 100 2026-02-22T23:11:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:11:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:11:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:11:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:11:15.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:11:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:11:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:11:15.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 11111.2 bytes/sec: 43 MiB/s 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T23:11:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:11:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:11:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:11:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:11:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:11:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:11:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:11:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:11:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:11:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:11:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:11:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T23:11:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:16.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:11:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:11:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:11:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:11:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:11:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:11:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:11:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:11:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:11:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:11:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:11:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:11:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T23:11:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:2 2026-02-22T23:11:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:11:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:11:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:11:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T23:11:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:11:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:11:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:11:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:16.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T23:11:16.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:16.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:3 2026-02-22T23:11:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:11:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:11:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:11:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:11:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:3 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:3 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:11:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T23:11:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:11:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:11:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:11:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:11:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T23:11:16.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:11:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:11:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:11:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T23:11:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:16.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:16.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:16.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:11:17.686 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:11:17.686 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:17.687 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:11:17.700 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T23:11:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:11:17.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:17.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:17.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:11:17.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:11:17.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:17.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:11:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:11:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:11:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:11:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:11:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:11:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:11:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:11:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:11:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:11:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:11:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:11:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:11:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:17.804 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:11:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:11:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:11:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:11:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:11:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_force_promote 2026-02-22T23:11:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote started 2026-02-22T23:11:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:11:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:11:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_force_promote started 2026-02-22T23:11:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:11:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:11:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:11:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:11:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:11:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:11:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:11:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:11:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:11:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:11:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:11:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:11:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:11:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:18.816 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:11:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T23:11:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:11:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:11:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:11:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:11:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:11:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:11:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:11:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:11:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:11:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:11:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:11:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:11:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_force_promote 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote 2026-02-22T23:11:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 149 2026-02-22T23:11:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_force_promote 2026-02-22T23:11:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:11:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_force_promote.status 2026-02-22T23:11:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_force_promote.status 2026-02-22T23:11:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_force_promote.status 2026-02-22T23:11:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_force_promote.status 2026-02-22T23:11:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_force_promote.status 2026-02-22T23:11:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_force_promote.status 2026-02-22T23:11:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_force_promote /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_force_promote.status 2026-02-22T23:11:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_force_promote.status 2026-02-22T23:11:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_force_promote --format xml 2026-02-22T23:11:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:11:19.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=149 2026-02-22T23:11:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:11:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:11:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:11:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_force_promote.status 2026-02-22T23:11:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_force_promote.status 2026-02-22T23:11:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote --format xml 2026-02-22T23:11:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:11:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=149 2026-02-22T23:11:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 149 = 149 2026-02-22T23:11:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_force_promote 2026-02-22T23:11:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote up+replaying 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:20.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote up+replaying '' '' 2026-02-22T23:11:20.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:20.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_force_promote: 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: c3ac37a3-ca45-4496-b917-0cf3f14c618d 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801875,"replay_state":"idle"} 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:17 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801875,"replay_state":"idle"} 2026-02-22T23:11:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:11:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:11:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:11:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:17' 2026-02-22T23:11:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote up+replaying '2026-02-22 23:11:17' 2026-02-22T23:11:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:11:17' 2026-02-22T23:11:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING1143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801867,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:11:08cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:11:15test_force_promotec3ac37a3-ca45-4496-b917-0cf3f14c618dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801875,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 23:11:17cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:11:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T23:11:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:17' 2026-02-22T23:11:20.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T23:11:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:11:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:11:20.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote up+stopped 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote up+stopped '' '' 2026-02-22T23:11:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_force_promote: 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: c3ac37a3-ca45-4496-b917-0cf3f14c618d 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:20 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801875,"replay_state":"idle"} 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:17 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 148 .mirror.primary.c3ac37a3-ca45-4496-b917-0cf3f14c618d.4a599d89-3370-40fa-9b62-9cdacfcc96d7 (peer_uuids:[]) 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 149 .mirror.primary.c3ac37a3-ca45-4496-b917-0cf3f14c618d.03c515cf-6c70-476c-afa3-045bc55ddb00 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:11:21.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:11:21.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:21.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:11:21.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:11:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:11:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:11:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:20' 2026-02-22T23:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote up+stopped '2026-02-22 23:11:20' 2026-02-22T23:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:11:20' 2026-02-22T23:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:21.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK244464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:11:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801867,"replay_state":"idle"}2026-02-22 23:11:08test_force_promotec3ac37a3-ca45-4496-b917-0cf3f14c618dup+stoppedlocal image is primary44534458mirror.1trial0822026-02-22 23:11:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801875,"replay_state":"idle"}2026-02-22 23:11:17' 2026-02-22T23:11:21.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:21.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T23:11:21.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:20' 2026-02-22T23:11:21.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T23:11:21.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:11:21.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:11:21.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:11:21.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:11:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test_force_promote --force 2026-02-22T23:11:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force=--force 2026-02-22T23:11:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_force_promote --force 2026-02-22T23:11:30.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:11:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test_force_promote 2026-02-22T23:11:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote stopped 2026-02-22T23:11:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:30.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:30.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:30.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:11:30.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:11:30.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:30.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_force_promote stopped 2026-02-22T23:11:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:11:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:11:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:11:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:11:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:11:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:11:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:11:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:11:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:11:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:11:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:11:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:11:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:11:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:11:31.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:31.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:31.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T23:11:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:11:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:11:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:31.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:11:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:11:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:11:31.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:11:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:11:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T23:11:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:11:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:11:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:11:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:11:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:11:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:11:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:11:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:11:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:11:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:11:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:11:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:11:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:11:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:32.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:11:32.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:32.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:32.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:11:32.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:32.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T23:11:32.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:32.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-02-22T23:11:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:11:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:11:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:11:32.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:11:32.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:11:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:11:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:11:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T23:11:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:11:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:11:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:11:32.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:11:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:11:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote up+stopped 2026-02-22T23:11:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote up+stopped '' '' 2026-02-22T23:11:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote 2026-02-22T23:11:33.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_force_promote: 2026-02-22T23:11:33.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: c3ac37a3-ca45-4496-b917-0cf3f14c618d 2026-02-22T23:11:33.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:33.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: force promoted 2026-02-22T23:11:33.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:11:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:30 2026-02-22T23:11:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:11:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:11:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:30 2026-02-22T23:11:33.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:11:33.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 150 .mirror.primary.c3ac37a3-ca45-4496-b917-0cf3f14c618d.7dd3a912-5293-4ec3-a633-26f4c247df73 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:11:33.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:11:33.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:11:33.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:33.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: force promoted 2026-02-22T23:11:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:11:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:11:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:11:33.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:33.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:33.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:30' 2026-02-22T23:11:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote up+stopped '2026-02-22 23:11:30' 2026-02-22T23:11:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:11:30' 2026-02-22T23:11:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK1143224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:11:23cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:11:30test_force_promotec3ac37a3-ca45-4496-b917-0cf3f14c618dup+stoppedforce promoted43284333mirror.1trial0822026-02-22 23:11:30cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:11:30' 2026-02-22T23:11:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T23:11:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:30' 2026-02-22T23:11:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T23:11:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:11:33.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:11:33.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:11:33.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:11:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote up+stopped 2026-02-22T23:11:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote up+stopped '' '' 2026-02-22T23:11:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote 2026-02-22T23:11:34.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_force_promote: 2026-02-22T23:11:34.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: c3ac37a3-ca45-4496-b917-0cf3f14c618d 2026-02-22T23:11:34.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:34.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:11:34.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:11:34.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:30 2026-02-22T23:11:34.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:34.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:11:34.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:34.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:11:34.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:33 2026-02-22T23:11:34.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:11:34.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 148 .mirror.primary.c3ac37a3-ca45-4496-b917-0cf3f14c618d.4a599d89-3370-40fa-9b62-9cdacfcc96d7 (peer_uuids:[]) 2026-02-22T23:11:34.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 149 .mirror.primary.c3ac37a3-ca45-4496-b917-0cf3f14c618d.03c515cf-6c70-476c-afa3-045bc55ddb00 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:11:34.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:11:34.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:11:34.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:34.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:11:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:11:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:11:34.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:34.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:11:34.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:30' 2026-02-22T23:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote up+stopped '2026-02-22 23:11:30' 2026-02-22T23:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:11:30' 2026-02-22T23:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:34.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK1144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:11:30cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}2026-02-22 23:11:23test_force_promotec3ac37a3-ca45-4496-b917-0cf3f14c618dup+stoppedlocal image is primary44534458mirror.1trial0822026-02-22 23:11:30cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:11:33' 2026-02-22T23:11:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:34.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T23:11:34.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:30' 2026-02-22T23:11:34.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T23:11:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:11:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:11:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:11:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:11:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster1 mirror test_force_promote 100 2026-02-22T23:11:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:11:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:11:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:11:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:11:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:11:34.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:11:34.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:11:34.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:11:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_force_promote 100 2026-02-22T23:11:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:11:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:11:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:11:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:11:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:11:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:11:34.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:11:34.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:11:34.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster1 mirror test_force_promote 2026-02-22T23:11:34.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:11:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:11:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster1 mirror test_force_promote 2026-02-22T23:11:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:34.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:34.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:34.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_force_promote 2026-02-22T23:11:34.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 rm mirror/test_force_promote 2026-02-22T23:11:35.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:11:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test_force_promote 2026-02-22T23:11:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:11:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:11:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test_force_promote 2026-02-22T23:11:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote 2026-02-22T23:11:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote 2026-02-22T23:11:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote 2026-02-22T23:11:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:11:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: cloned images' 2026-02-22T23:11:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:11:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:11:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:11:36 TEST: cloned images 2026-02-22T23:11:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:11:36 TEST: cloned images 2026-02-22T23:11:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog ' - default' 2026-02-22T23:11:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:11:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:11:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:11:36 - default 2026-02-22T23:11:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:11:36 - default 2026-02-22T23:11:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ parent_image=test_parent 2026-02-22T23:11:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ parent_snap=snap 2026-02-22T23:11:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror_parent test_parent snapshot 2026-02-22T23:11:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror_parent test_parent 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror_parent/test_parent 2026-02-22T23:11:36.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror_parent 2026-02-22T23:11:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:11:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror_parent test_parent snapshot 2026-02-22T23:11:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:11:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror_parent/test_parent snapshot 2026-02-22T23:11:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:11:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror_parent/test_parent 2026-02-22T23:11:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_parent': 2026-02-22T23:11:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:11:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:11:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:11:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 1524dddec9d3 2026-02-22T23:11:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.1524dddec9d3 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:11:36 2026 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:11:36 2026 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:11:36 2026 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: fb2cb35b-6dd3-46bb-bff6-6467d0232d16 2026-02-22T23:11:37.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:11:37.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror_parent test_parent 100 2026-02-22T23:11:37.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:11:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:11:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:11:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:11:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror_parent/test_parent --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:11:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:11:37.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:11:37.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 12500.1 bytes/sec: 49 MiB/s 2026-02-22T23:11:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T23:11:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap 2026-02-22T23:11:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap 2026-02-22T23:11:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:11:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ protect_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T23:11:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap 2026-02-22T23:11:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap protect mirror_parent/test_parent@snap 2026-02-22T23:11:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ clone_image=test_clone 2026-02-22T23:11:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ clone_image cluster2 mirror_parent test_parent snap mirror test_clone 2026-02-22T23:11:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_pool=mirror_parent 2026-02-22T23:11:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_image=test_parent 2026-02-22T23:11:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_snap=snap 2026-02-22T23:11:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_pool=mirror 2026-02-22T23:11:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_image=test_clone 2026-02-22T23:11:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 6 2026-02-22T23:11:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap mirror/test_clone --image-feature layering,exclusive-lock 2026-02-22T23:11:38.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_clone 100 2026-02-22T23:11:38.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:38.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:38.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:38.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:11:38.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:11:38.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:11:38.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:11:38.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_clone --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:11:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:11:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 6666.74 bytes/sec: 26 MiB/s 2026-02-22T23:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test_clone snapshot 2026-02-22T23:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:11:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone snapshot 2026-02-22T23:11:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:11:39.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test_clone 2026-02-22T23:11:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_clone': 2026-02-22T23:11:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:11:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:11:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:11:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 15368ffa2966 2026-02-22T23:11:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.15368ffa2966 2026-02-22T23:11:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:11:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:11:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:11:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:11:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:11:38 2026 2026-02-22T23:11:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:11:38 2026 2026-02-22T23:11:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:11:38 2026 2026-02-22T23:11:39.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: parent: mirror_parent/test_parent@snap 2026-02-22T23:11:39.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: overlap: 128 MiB 2026-02-22T23:11:39.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:11:39.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:11:39.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: edb49021-561a-4f9d-91b4-2104b5d81a57 2026-02-22T23:11:39.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:11:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror_parent test_parent 2026-02-22T23:11:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror_parent test_parent started 2026-02-22T23:11:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:11:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:11:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror_parent test_parent started 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror_parent/test_parent 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:11:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:11:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/test_parent 2026-02-22T23:11:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:11:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:11:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:11:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/test_parent 2026-02-22T23:11:40.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:11:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:11:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:11:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:11:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:11:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:11:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:11:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror_parent test_parent 2026-02-22T23:11:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:11:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:11:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:11:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-22T23:11:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-22T23:11:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 5 2026-02-22T23:11:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror_parent test_parent 2026-02-22T23:11:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:11:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror_parent-test_parent.status 2026-02-22T23:11:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror_parent-test_parent.status 2026-02-22T23:11:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror_parent-test_parent.status 2026-02-22T23:11:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror_parent test_parent /tmp/tmp.VQMrvUqClS/cluster2-mirror_parent-test_parent.status 2026-02-22T23:11:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror_parent-test_parent.status 2026-02-22T23:11:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror_parent/test_parent --format xml 2026-02-22T23:11:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:11:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=5 2026-02-22T23:11:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:11:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:11:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:11:41.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:41.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:41.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:41.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:41.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:41.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-22T23:11:41.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:11:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=3 2026-02-22T23:11:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 5 = 3 2026-02-22T23:11:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:11:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:11:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-22T23:11:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:41.740 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:11:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=3 2026-02-22T23:11:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 5 = 3 2026-02-22T23:11:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:11:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T23:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-22T23:11:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:11:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:11:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:11:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:11:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:11:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:11:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 5 = '' 2026-02-22T23:11:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:11:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1.6 2026-02-22T23:11:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.status 2026-02-22T23:11:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-22T23:11:44.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=5 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 5 = 5 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror_parent test_parent 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror_parent test_parent up+replaying 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror_parent test_parent up+replaying '' '' 2026-02-22T23:11:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:45.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:45.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:45.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:45.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror_parent/test_parent 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_parent: 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: fb2cb35b-6dd3-46bb-bff6-6467d0232d16 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801897,"replay_state":"idle"} 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:39 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:40' 2026-02-22T23:11:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:11:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:11:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801897,"replay_state":"idle"} 2026-02-22T23:11:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:11:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:11:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:11:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:39' 2026-02-22T23:11:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror_parent test_parent up+replaying '2026-02-22 23:11:39' 2026-02-22T23:11:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:11:39' 2026-02-22T23:11:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:45.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format xml 2026-02-22T23:11:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK143224342mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284348mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354352mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434354mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_parentfb2cb35b-6dd3-46bb-bff6-6467d0232d16up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801897,"replay_state":"idle"}43224342mirror.0trial0822026-02-22 23:11:39cluster2-DNEf95547cc-a01e-4e8f-ae90-5d2a2f0ab001up+stoppedlocal image is primary2026-02-22 23:11:40' 2026-02-22T23:11:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_parent'\'']/last_update' 2026-02-22T23:11:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:39' 2026-02-22T23:11:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_parent'\'']/state' 2026-02-22T23:11:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:11:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:11:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:11:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:11:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror_parent test_parent 2026-02-22T23:11:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:11:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror_parent-test_parent.export 2026-02-22T23:11:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror_parent-test_parent.export 2026-02-22T23:11:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror_parent-test_parent.export 2026-02-22T23:11:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror_parent-test_parent.export 2026-02-22T23:11:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror_parent-test_parent.export 2026-02-22T23:11:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.export 2026-02-22T23:11:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror_parent-test_parent.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.export 2026-02-22T23:11:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror_parent/test_parent /tmp/tmp.VQMrvUqClS/cluster2-mirror_parent-test_parent.export 2026-02-22T23:11:45.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:11:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror_parent/test_parent /tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.export 2026-02-22T23:11:45.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:11:45.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror_parent-test_parent.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.export 2026-02-22T23:11:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror_parent-test_parent.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_parent-test_parent.export 2026-02-22T23:11:45.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone 2026-02-22T23:11:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone started 2026-02-22T23:11:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:11:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:11:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_clone started 2026-02-22T23:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:11:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone 2026-02-22T23:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:11:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:11:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone 2026-02-22T23:11:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:11:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:11:46.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:46.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:46.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:46.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone 2026-02-22T23:11:46.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone'] 2026-02-22T23:11:46.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:46.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:46.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:46.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone 2026-02-22T23:11:46.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:11:46.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:11:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:46.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:46.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:11:46.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:11:46.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:11:46.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone 2026-02-22T23:11:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:11:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:11:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:11:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:11:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_clone 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test_clone 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_clone 2026-02-22T23:11:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 151 2026-02-22T23:11:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_clone 2026-02-22T23:11:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:11:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_clone.status 2026-02-22T23:11:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_clone.status 2026-02-22T23:11:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_clone.status 2026-02-22T23:11:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_clone.status 2026-02-22T23:11:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_clone.status 2026-02-22T23:11:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.status 2026-02-22T23:11:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_clone /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_clone.status 2026-02-22T23:11:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_clone.status 2026-02-22T23:11:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_clone --format xml 2026-02-22T23:11:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:47.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:11:47.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=151 2026-02-22T23:11:47.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:11:47.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:11:47.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:11:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_clone /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.status 2026-02-22T23:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.status 2026-02-22T23:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_clone --format xml 2026-02-22T23:11:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:11:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=150 2026-02-22T23:11:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 151 = 150 2026-02-22T23:11:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:11:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:11:47.695 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_clone /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.status 2026-02-22T23:11:47.695 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:47.695 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.status 2026-02-22T23:11:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_clone --format xml 2026-02-22T23:11:47.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:11:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=150 2026-02-22T23:11:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 151 = 150 2026-02-22T23:11:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:11:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T23:11:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_clone /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.status 2026-02-22T23:11:48.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:48.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:48.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:48.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.status 2026-02-22T23:11:48.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_clone --format xml 2026-02-22T23:11:48.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:11:48.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:11:48.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=151 2026-02-22T23:11:48.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 151 = 151 2026-02-22T23:11:48.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_clone 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_clone up+replaying 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:49.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_clone up+replaying '' '' 2026-02-22T23:11:49.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:49.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:49.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:11:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:11:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_clone 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_clone: 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: edb49021-561a-4f9d-91b4-2104b5d81a57 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801899,"replay_state":"idle"} 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:44 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:11:45' 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:11:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:11:49.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801899,"replay_state":"idle"} 2026-02-22T23:11:49.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:11:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:11:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:11:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:11:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:11:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:11:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:11:49.612 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:11:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:44' 2026-02-22T23:11:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_clone up+replaying '2026-02-22 23:11:44' 2026-02-22T23:11:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:11:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:11:44' 2026-02-22T23:11:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:11:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:11:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:11:49.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:11:23cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:11:45test_cloneedb49021-561a-4f9d-91b4-2104b5d81a57up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801899,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 23:11:44cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:11:45' 2026-02-22T23:11:49.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:11:49.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_clone'\'']/last_update' 2026-02-22T23:11:49.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:11:44' 2026-02-22T23:11:49.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_clone'\'']/state' 2026-02-22T23:11:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:11:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:11:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:11:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:11:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test_clone 2026-02-22T23:11:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:11:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_clone.export 2026-02-22T23:11:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_clone.export 2026-02-22T23:11:49.656 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_clone.export 2026-02-22T23:11:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_clone.export 2026-02-22T23:11:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_clone.export 2026-02-22T23:11:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:11:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.export 2026-02-22T23:11:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_clone.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.export 2026-02-22T23:11:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test_clone /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_clone.export 2026-02-22T23:11:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:11:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test_clone /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.export 2026-02-22T23:11:49.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:11:49.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_clone.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.export 2026-02-22T23:11:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_clone.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_clone.export 2026-02-22T23:11:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test_clone 2026-02-22T23:11:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:11:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:11:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test_clone 2026-02-22T23:11:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone 2026-02-22T23:11:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone 2026-02-22T23:11:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone 2026-02-22T23:11:50.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:11:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog ' - clone v1' 2026-02-22T23:11:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:11:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:11:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:11:50 - clone v1 2026-02-22T23:11:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:11:50 - clone v1 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ clone_image_and_enable_mirror cluster1 mirror_parent test_parent snap mirror test_clone1 snapshot 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_pool=mirror_parent 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_image=test_parent 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_snap=snap 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_pool=mirror 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_image=test_clone1 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 6 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ clone_image cluster1 mirror_parent test_parent snap mirror test_clone1 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_pool=mirror_parent 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_image=test_parent 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_snap=snap 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_pool=mirror 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_image=test_clone1 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 6 2026-02-22T23:11:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 clone mirror_parent/test_parent@snap mirror/test_clone1 --image-feature layering,exclusive-lock 2026-02-22T23:11:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:11:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:11:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster1 mirror test_clone1 snapshot 2026-02-22T23:11:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone1 2026-02-22T23:11:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:11:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/test_clone1 snapshot 2026-02-22T23:11:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:11:51.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 info mirror/test_clone1 2026-02-22T23:11:51.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_clone1': 2026-02-22T23:11:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:11:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:11:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:11:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 1556b43728ad 2026-02-22T23:11:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.1556b43728ad 2026-02-22T23:11:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:11:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:11:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:11:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:11:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:11:50 2026 2026-02-22T23:11:51.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:11:50 2026 2026-02-22T23:11:51.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:11:50 2026 2026-02-22T23:11:51.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: parent: mirror_parent/test_parent@snap 2026-02-22T23:11:51.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: overlap: 128 MiB 2026-02-22T23:11:51.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:11:51.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:11:51.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 7d26fe67-d76c-4849-8aa9-725a494283f3 2026-02-22T23:11:51.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:11:51.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ clone_image_and_enable_mirror cluster2 mirror_parent test_parent snap mirror test_clone_v1 snapshot --rbd-default-clone-format 1 2026-02-22T23:11:51.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:51.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_pool=mirror_parent 2026-02-22T23:11:51.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_image=test_parent 2026-02-22T23:11:51.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_snap=snap 2026-02-22T23:11:51.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_pool=mirror 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_image=test_clone_v1 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 6 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ clone_image cluster2 mirror_parent test_parent snap mirror test_clone_v1 --rbd-default-clone-format 1 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_pool=mirror_parent 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_image=test_parent 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_snap=snap 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_pool=mirror 2026-02-22T23:11:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_image=test_clone_v1 2026-02-22T23:11:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 6 2026-02-22T23:11:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap mirror/test_clone_v1 --image-feature layering,exclusive-lock --rbd-default-clone-format 1 2026-02-22T23:11:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:11:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:11:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test_clone_v1 snapshot 2026-02-22T23:11:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v1 2026-02-22T23:11:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:11:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone_v1 snapshot 2026-02-22T23:11:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:11:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test_clone_v1 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_clone_v1': 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 155afe69ca14 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.155afe69ca14 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:11:51 2026 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:11:51 2026 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:11:51 2026 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: parent: mirror_parent/test_parent@snap 2026-02-22T23:11:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: overlap: 128 MiB 2026-02-22T23:11:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:11:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:11:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: f3673fc6-5f6f-4a32-a825-7593e86dca0f 2026-02-22T23:11:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:11:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_clone_format cluster2 mirror test_clone_v1 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_clone_v1 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 info mirror/test_clone_v1 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ awk 'BEGIN { 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: format = 1 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: $1 == "parent:" { 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: parent = $2 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: /op_features: .*clone-child/ { 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: format = 2 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: END { 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: if (!parent) exit 1 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: print format 2026-02-22T23:11:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: }' 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 1 = 1 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone_v1 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v1 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone_v1 started 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v1 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:52.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:11:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_clone_v1 started 2026-02-22T23:11:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:53.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v1 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:11:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:53.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:11:53.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:11:53.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:53.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:11:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:11:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:11:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:11:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:11:53.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:53.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:53.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:53.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T23:11:53.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:11:53.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:53.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:53.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:11:53.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:11:53.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:11:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T23:11:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:11:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:11:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:11:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:11:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:11:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:11:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:11:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:11:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:11:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:53.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T23:11:53.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:53.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:11:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:11:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:11:53.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:11:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:11:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:11:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:11:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:11:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T23:11:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:11:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:11:54.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:11:54.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:11:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:11:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:11:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T23:11:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:11:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:11:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:11:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:11:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:11:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T23:11:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:11:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:11:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:11:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:11:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:11:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:11:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:11:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:11:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:11:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:11:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:11:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:11:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:11:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:11:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:11:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:11:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:11:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:11:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:11:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:11:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:11:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:11:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:11:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:11:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:11:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v1 2026-02-22T23:11:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:11:54.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:11:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:11:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:11:54.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:11:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:11:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:11:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_clone_format cluster1 mirror test_clone_v1 2026-02-22T23:11:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:11:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:11:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_clone_v1 2026-02-22T23:11:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 info mirror/test_clone_v1 2026-02-22T23:11:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ awk 'BEGIN { 2026-02-22T23:11:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: format = 1 2026-02-22T23:11:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:11:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: $1 == "parent:" { 2026-02-22T23:11:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: parent = $2 2026-02-22T23:11:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:11:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: /op_features: .*clone-child/ { 2026-02-22T23:11:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: format = 2 2026-02-22T23:11:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:11:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: END { 2026-02-22T23:11:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: if (!parent) exit 1 2026-02-22T23:11:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: print format 2026-02-22T23:11:54.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: }' 2026-02-22T23:11:54.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 1 = 1 2026-02-22T23:11:54.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test_clone_v1 2026-02-22T23:11:54.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:54.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:54.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v1 2026-02-22T23:11:54.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:11:54.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:11:54.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test_clone_v1 2026-02-22T23:11:54.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:54.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:54.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v1 2026-02-22T23:11:54.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone_v1 2026-02-22T23:11:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone_v1 2026-02-22T23:11:55.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:11:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster1 mirror test_clone1 2026-02-22T23:11:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone1 2026-02-22T23:11:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:11:55.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:11:55.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster1 mirror test_clone1 2026-02-22T23:11:55.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:11:55.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:11:55.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone1 2026-02-22T23:11:55.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_clone1 2026-02-22T23:11:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 rm mirror/test_clone1 2026-02-22T23:11:56.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:11:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ unprotect_snapshot_retry cluster2 mirror_parent test_parent snap 2026-02-22T23:11:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap 2026-02-22T23:11:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:11:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:11:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ unprotect_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T23:11:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap 2026-02-22T23:11:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap unprotect mirror_parent/test_parent@snap 2026-02-22T23:11:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:11:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T23:11:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:56.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:56.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap 2026-02-22T23:11:56.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap 2026-02-22T23:11:57.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing snap: 100% complete...done. 2026-02-22T23:11:57.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog ' - clone v2' 2026-02-22T23:11:57.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:11:57.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:11:57.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:11:57 - clone v2 2026-02-22T23:11:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:11:57 - clone v2 2026-02-22T23:11:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ parent_snap=snap_v2 2026-02-22T23:11:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T23:11:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_v2 2026-02-22T23:11:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap_v2 2026-02-22T23:11:58.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:11:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:11:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-22T23:11:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:11:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:11:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:11:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-22T23:12:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 7 2026-02-22T23:12:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ clone_image_and_enable_mirror cluster2 mirror_parent test_parent snap_v2 mirror test_clone_v2 snapshot --rbd-default-clone-format 2 2026-02-22T23:12:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_pool=mirror_parent 2026-02-22T23:12:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_image=test_parent 2026-02-22T23:12:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_snap=snap_v2 2026-02-22T23:12:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_pool=mirror 2026-02-22T23:12:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_image=test_clone_v2 2026-02-22T23:12:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 6 2026-02-22T23:12:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:12:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ clone_image cluster2 mirror_parent test_parent snap_v2 mirror test_clone_v2 --rbd-default-clone-format 2 2026-02-22T23:12:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_pool=mirror_parent 2026-02-22T23:12:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_image=test_parent 2026-02-22T23:12:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_snap=snap_v2 2026-02-22T23:12:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_pool=mirror 2026-02-22T23:12:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_image=test_clone_v2 2026-02-22T23:12:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 6 2026-02-22T23:12:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap_v2 mirror/test_clone_v2 --image-feature layering,exclusive-lock --rbd-default-clone-format 2 2026-02-22T23:12:00.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:12:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:12:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test_clone_v2 snapshot 2026-02-22T23:12:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone_v2 snapshot 2026-02-22T23:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:12:01.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test_clone_v2 2026-02-22T23:12:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_clone_v2': 2026-02-22T23:12:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:12:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:12:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:12:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 15788b4e67c2 2026-02-22T23:12:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.15788b4e67c2 2026-02-22T23:12:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:12:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, operations 2026-02-22T23:12:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: clone-child 2026-02-22T23:12:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:12:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:12:00 2026 2026-02-22T23:12:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:12:00 2026 2026-02-22T23:12:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:12:00 2026 2026-02-22T23:12:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: parent: mirror_parent/test_parent@snap_v2 2026-02-22T23:12:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: overlap: 128 MiB 2026-02-22T23:12:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:12:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:12:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 46951515-628c-404e-8ef2-20f7e6868df0 2026-02-22T23:12:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:12:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_clone_format cluster2 mirror test_clone_v2 2026-02-22T23:12:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:12:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_clone_v2 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 info mirror/test_clone_v2 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ awk 'BEGIN { 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: format = 1 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: $1 == "parent:" { 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: parent = $2 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: /op_features: .*clone-child/ { 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: format = 2 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: END { 2026-02-22T23:12:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: if (!parent) exit 1 2026-02-22T23:12:01.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: print format 2026-02-22T23:12:01.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: }' 2026-02-22T23:12:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 2 = 2 2026-02-22T23:12:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone_v2 2026-02-22T23:12:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone_v2 started 2026-02-22T23:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:01.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:12:01.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_clone_v2 started 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:12:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:12:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:12:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:12:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:12:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:12:02.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:12:02.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:02.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:02.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:02.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T23:12:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:12:02.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:12:02.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:12:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:12:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T23:12:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:12:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:12:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:12:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:12:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:12:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:12:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:12:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:12:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:12:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:12:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T23:12:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:12:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:12:02.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:02.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:02.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:12:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:12:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:12:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:12:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:12:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:12:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:12:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:12:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:12:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:12:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:12:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:12:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:12:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:12:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:12:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:12:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:12:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:12:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:12:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T23:12:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:12:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:12:02.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:12:02.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:12:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:12:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:12:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:12:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:12:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:12:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:12:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:12:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:12:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:12:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:12:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:12:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:12:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:12:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:12:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:12:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:12:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:12:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:12:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T23:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:12:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:12:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:12:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:03.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:12:03.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:12:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:12:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:12:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:12:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:12:03.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-02-22T23:12:03.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:12:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:12:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:12:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:12:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:12:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:12:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:12:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_clone_format cluster1 mirror test_clone_v2 2026-02-22T23:12:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_clone_v2 2026-02-22T23:12:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 info mirror/test_clone_v2 2026-02-22T23:12:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ awk 'BEGIN { 2026-02-22T23:12:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: format = 1 2026-02-22T23:12:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:12:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: $1 == "parent:" { 2026-02-22T23:12:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: parent = $2 2026-02-22T23:12:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:12:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: /op_features: .*clone-child/ { 2026-02-22T23:12:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: format = 2 2026-02-22T23:12:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: } 2026-02-22T23:12:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: END { 2026-02-22T23:12:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: if (!parent) exit 1 2026-02-22T23:12:03.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: print format 2026-02-22T23:12:03.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: }' 2026-02-22T23:12:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 2 = 2 2026-02-22T23:12:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T23:12:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_v2 2026-02-22T23:12:03.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap_v2 2026-02-22T23:12:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing snap: 100% complete...done. 2026-02-22T23:12:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:12:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-22T23:12:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:03.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-22T23:12:03.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 8 2026-02-22T23:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T23:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_v2 2026-02-22T23:12:03.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-22T23:12:03.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T23:12:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 6 256a9310-9365-4b0b-9dd6-de9f87f396e2 128 MiB Sun Feb 22 23:11:58 2026 trash (user snap_v2) 2026-02-22T23:12:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:12:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_v2 2026-02-22T23:12:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T23:12:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T23:12:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ continue 2026-02-22T23:12:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:12:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_v2 2026-02-22T23:12:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T23:12:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T23:12:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ continue 2026-02-22T23:12:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:12:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_v2 2026-02-22T23:12:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T23:12:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T23:12:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 6 5c7c6bd0-3702-4466-9e0c-1cee36516b35 128 MiB Sun Feb 22 23:11:59 2026 trash (user snap_v2) 2026-02-22T23:12:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test_clone_v2 2026-02-22T23:12:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:12:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:12:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test_clone_v2 2026-02-22T23:12:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone_v2 2026-02-22T23:12:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone_v2 2026-02-22T23:12:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test_clone_v2 deleted 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test_clone_v2 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_clone_v2 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-22T23:12:12.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:12:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test_clone_v2: (2) No such file or directory 2026-02-22T23:12:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id= 2026-02-22T23:12:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:12:12.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:12:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test_clone_v2 deleted '' 2026-02-22T23:12:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:12:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:12:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:12:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:12:12.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test_clone_v2 2026-02-22T23:12:12.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:12.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:12.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_clone_v2 2026-02-22T23:12:12.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-22T23:12:12.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:12:12.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test_clone_v2: (2) No such file or directory 2026-02-22T23:12:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:12:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:12:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_removed_from_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T23:12:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T23:12:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_v2 2026-02-22T23:12:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-22T23:12:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T23:12:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:12:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_v2 2026-02-22T23:12:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T23:12:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T23:12:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog ' - clone v2 non-primary' 2026-02-22T23:12:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:12:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:12:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:12:13 - clone v2 non-primary 2026-02-22T23:12:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:12:13 - clone v2 non-primary 2026-02-22T23:12:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T23:12:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_v2 2026-02-22T23:12:13.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap_v2 2026-02-22T23:12:14.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:12:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:12:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-22T23:12:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:14.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-22T23:12:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 10 2026-02-22T23:12:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_present cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_v2 2026-02-22T23:12:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:12:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror_parent/test_parent@snap_v2 2026-02-22T23:12:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_parent': 2026-02-22T23:12:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:12:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 2 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 10f65275842d 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.10f65275842d 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:11:39 2026 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:11:39 2026 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:11:39 2026 2026-02-22T23:12:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: protected: False 2026-02-22T23:12:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:12:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:12:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: fb2cb35b-6dd3-46bb-bff6-6467d0232d16 2026-02-22T23:12:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: false 2026-02-22T23:12:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ clone_image_and_enable_mirror cluster1 mirror_parent test_parent snap_v2 mirror test_clone_v2 snapshot --rbd-default-clone-format 2 2026-02-22T23:12:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_pool=mirror_parent 2026-02-22T23:12:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_image=test_parent 2026-02-22T23:12:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_snap=snap_v2 2026-02-22T23:12:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_pool=mirror 2026-02-22T23:12:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_image=test_clone_v2 2026-02-22T23:12:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 6 2026-02-22T23:12:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ clone_image cluster1 mirror_parent test_parent snap_v2 mirror test_clone_v2 --rbd-default-clone-format 2 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_pool=mirror_parent 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_image=test_parent 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local parent_snap=snap_v2 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_pool=mirror 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local clone_image=test_clone_v2 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 6 2026-02-22T23:12:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 clone mirror_parent/test_parent@snap_v2 mirror/test_clone_v2 --image-feature layering,exclusive-lock --rbd-default-clone-format 2 2026-02-22T23:12:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:12:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:12:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster1 mirror test_clone_v2 snapshot 2026-02-22T23:12:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/test_clone_v2 snapshot 2026-02-22T23:12:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:12:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-22T23:12:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_clone_v2': 2026-02-22T23:12:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:12:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:12:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:12:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 1579f4a95948 2026-02-22T23:12:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.1579f4a95948 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, operations 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: clone-child 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:12:18 2026 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:12:18 2026 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:12:18 2026 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: parent: mirror_parent/test_parent@snap_v2 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: overlap: 128 MiB 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:12:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 7275b779-f1cd-4251-86ca-842b6f395bf5 2026-02-22T23:12:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:12:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T23:12:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_v2 2026-02-22T23:12:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap_v2 2026-02-22T23:12:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing snap: 100% complete...done. 2026-02-22T23:12:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_removed_from_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T23:12:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T23:12:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_v2 2026-02-22T23:12:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-22T23:12:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T23:12:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:12:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-22T23:12:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-22T23:12:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 11 2026-02-22T23:12:22.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:12:23.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:23.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:23.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:23.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:23.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_v2 2026-02-22T23:12:23.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T23:12:23.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T23:12:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 9 7ec15316-d730-4612-910a-af5e5105d6e6 128 MiB Sun Feb 22 23:12:16 2026 trash (user snap_v2) 2026-02-22T23:12:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster1 mirror test_clone_v2 2026-02-22T23:12:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:12:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:12:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster1 mirror test_clone_v2 2026-02-22T23:12:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:23.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_clone_v2 2026-02-22T23:12:23.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_clone_v2 2026-02-22T23:12:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 rm mirror/test_clone_v2 2026-02-22T23:12:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:12:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:12:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T23:12:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_v2 2026-02-22T23:12:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T23:12:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T23:12:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror_parent test_parent 2026-02-22T23:12:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:12:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:12:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror_parent test_parent 2026-02-22T23:12:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror_parent 2026-02-22T23:12:26.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_parent 2026-02-22T23:12:26.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror_parent/test_parent 2026-02-22T23:12:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror_parent/test_parent 2026-02-22T23:12:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:12:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: data pool' 2026-02-22T23:12:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:12:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:12:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:12:27 TEST: data pool 2026-02-22T23:12:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:12:27 TEST: data pool 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ dp_image=test_data_pool 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test_data_pool snapshot 128 --data-pool mirror_parent 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test_data_pool 128 --data-pool mirror_parent 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:12:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 128 ']' 2026-02-22T23:12:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=128 2026-02-22T23:12:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock --data-pool mirror_parent mirror/test_data_pool 2026-02-22T23:12:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:12:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:12:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test_data_pool snapshot 2026-02-22T23:12:27.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:27.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:27.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:27.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:27.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_data_pool snapshot 2026-02-22T23:12:28.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:12:28.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test_data_pool 2026-02-22T23:12:28.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_data_pool': 2026-02-22T23:12:28.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:12:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:12:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:12:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 15abf32d10c3 2026-02-22T23:12:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: data_pool: mirror_parent 2026-02-22T23:12:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.3.15abf32d10c3 2026-02-22T23:12:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:12:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, data-pool 2026-02-22T23:12:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:12:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:12:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:12:27 2026 2026-02-22T23:12:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:12:27 2026 2026-02-22T23:12:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:12:27 2026 2026-02-22T23:12:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:12:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:12:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: d987bb36-031b-4c7c-b213-85233162df6e 2026-02-22T23:12:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:12:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_data_pool cluster2 mirror test_data_pool 2026-02-22T23:12:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:12:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_data_pool 2026-02-22T23:12:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 info mirror/test_data_pool 2026-02-22T23:12:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T23:12:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ data_pool=mirror_parent 2026-02-22T23:12:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test mirror_parent = mirror_parent 2026-02-22T23:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_data_pool 2026-02-22T23:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_data_pool started 2026-02-22T23:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:12:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_data_pool started 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_data_pool 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:12:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:12:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:12:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:12:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:12:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:12:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-02-22T23:12:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T23:12:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:12:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:12:29.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:29.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_data_pool 2026-02-22T23:12:29.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:12:29.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:29.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:29.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:12:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:12:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:12:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:12:29.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:29.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:29.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:29.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_data_pool 2026-02-22T23:12:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T23:12:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:12:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:12:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_data_pool 2026-02-22T23:12:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:12:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:12:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:12:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:12:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:12:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:12:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:12:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:12:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:12:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:12:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:12:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:12:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:12:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:12:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:12:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:12:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:12:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:12:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:12:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_data_pool 2026-02-22T23:12:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T23:12:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:12:29.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_data_pool 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:12:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:12:29.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:12:29.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:12:29.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:12:29.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:12:29.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_data_pool 2026-02-22T23:12:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T23:12:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:12:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:12:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:12:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:12:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:12:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:12:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-02-22T23:12:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:12:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:12:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:12:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:12:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:12:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:12:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-02-22T23:12:30.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T23:12:30.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:12:30.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_data_pool 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:12:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:12:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:12:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:12:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_data_pool 2026-02-22T23:12:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:12:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:12:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:12:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:12:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:12:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:12:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:12:30.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:30.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_data_pool cluster1 mirror test_data_pool 2026-02-22T23:12:30.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:30.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:30.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_data_pool 2026-02-22T23:12:30.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 info mirror/test_data_pool 2026-02-22T23:12:30.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T23:12:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ data_pool=mirror_parent 2026-02-22T23:12:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test mirror_parent = mirror_parent 2026-02-22T23:12:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test_data_pool snap1 2026-02-22T23:12:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:12:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test_data_pool@snap1 2026-02-22T23:12:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:12:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_data_pool 100 2026-02-22T23:12:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:12:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:12:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:31.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:12:31.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:12:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:12:31.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:12:31.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 11111.2 bytes/sec: 43 MiB/s 2026-02-22T23:12:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test_data_pool snap2 2026-02-22T23:12:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:12:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test_data_pool@snap2 2026-02-22T23:12:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:12:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_data_pool 100 2026-02-22T23:12:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:12:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:12:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:12:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:12:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:12:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:12:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 5555.61 bytes/sec: 22 MiB/s 2026-02-22T23:12:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_data_pool 2026-02-22T23:12:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:12:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:12:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:12:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test_data_pool 2026-02-22T23:12:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:32.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_data_pool 2026-02-22T23:12:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 15 2026-02-22T23:12:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_data_pool 2026-02-22T23:12:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:12:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T23:12:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_data_pool.status 2026-02-22T23:12:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:12:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool.status 2026-02-22T23:12:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_data_pool.status 2026-02-22T23:12:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_data_pool.status 2026-02-22T23:12:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:12:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.status 2026-02-22T23:12:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_data_pool /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool.status 2026-02-22T23:12:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool.status 2026-02-22T23:12:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_data_pool --format xml 2026-02-22T23:12:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:12:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:12:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=15 2026-02-22T23:12:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:12:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:12:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:12:33.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.status 2026-02-22T23:12:33.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:33.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:33.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:33.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.status 2026-02-22T23:12:33.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-22T23:12:33.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:12:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:12:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=12 2026-02-22T23:12:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 15 = 12 2026-02-22T23:12:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:12:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:12:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.status 2026-02-22T23:12:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.status 2026-02-22T23:12:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-22T23:12:33.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:12:33.741 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:12:33.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=12 2026-02-22T23:12:33.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 15 = 12 2026-02-22T23:12:33.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:12:33.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T23:12:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.status 2026-02-22T23:12:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.status 2026-02-22T23:12:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-22T23:12:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:12:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:12:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:12:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:12:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:12:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:12:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:12:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 15 = '' 2026-02-22T23:12:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:12:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1.6 2026-02-22T23:12:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.status 2026-02-22T23:12:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.status 2026-02-22T23:12:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-22T23:12:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:12:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:12:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=15 2026-02-22T23:12:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 15 = 15 2026-02-22T23:12:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_data_pool 2026-02-22T23:12:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:12:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_data_pool up+replaying 2026-02-22T23:12:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:12:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:12:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:12:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:12:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:12:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_data_pool up+replaying '' '' 2026-02-22T23:12:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:12:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:12:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:12:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:12:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:12:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_data_pool 2026-02-22T23:12:37.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_data_pool: 2026-02-22T23:12:37.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d987bb36-031b-4c7c-b213-85233162df6e 2026-02-22T23:12:37.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:12:37.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801948,"replay_state":"idle"} 2026-02-22T23:12:37.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:12:37.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:12:30 2026-02-22T23:12:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:12:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:12:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:12:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:12:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:12:35' 2026-02-22T23:12:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:12:37.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:12:37.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:12:37.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801948,"replay_state":"idle"} 2026-02-22T23:12:37.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:12:37.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:12:37.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:12:37.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:12:37.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:12:37.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:12:37.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:12:37.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:12:37.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:12:30' 2026-02-22T23:12:37.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_data_pool up+replaying '2026-02-22 23:12:30' 2026-02-22T23:12:37.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:37.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:12:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:12:30' 2026-02-22T23:12:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:12:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:12:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:12:37.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:12:23cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:12:35test_data_poold987bb36-031b-4c7c-b213-85233162df6eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771801948,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 23:12:30cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:12:35' 2026-02-22T23:12:37.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:12:37.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/last_update' 2026-02-22T23:12:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:12:30' 2026-02-22T23:12:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/state' 2026-02-22T23:12:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:12:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:12:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:12:37.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:12:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test_data_pool@snap1 2026-02-22T23:12:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool@snap1 2026-02-22T23:12:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:12:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:12:37.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:12:37.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool@snap1 /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:12:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool@snap1 /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:12:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T23:12:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test_data_pool@snap2 2026-02-22T23:12:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool@snap2 2026-02-22T23:12:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:12:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:12:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:12:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool@snap2 /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:12:37.564 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool@snap2 /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:12:37.654 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T23:12:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test_data_pool 2026-02-22T23:12:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:12:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_data_pool.export 2026-02-22T23:12:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_data_pool.export 2026-02-22T23:12:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:12:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool.export 2026-02-22T23:12:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_data_pool.export 2026-02-22T23:12:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_data_pool.export 2026-02-22T23:12:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:12:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.export 2026-02-22T23:12:37.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.export 2026-02-22T23:12:37.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool.export 2026-02-22T23:12:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:12:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.export 2026-02-22T23:12:37.922 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:12:37.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.export 2026-02-22T23:12:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_data_pool.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_data_pool.export 2026-02-22T23:12:38.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:38.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test_data_pool 2026-02-22T23:12:38.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:38.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:38.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:38.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:12:38.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:12:38.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test_data_pool 2026-02-22T23:12:38.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:38.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:38.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:12:38.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_data_pool 2026-02-22T23:12:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T23:12:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test_data_pool 2026-02-22T23:12:41.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:12:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: disable mirroring / delete non-primary image' 2026-02-22T23:12:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:12:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:12:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:12:41 TEST: disable mirroring / delete non-primary image 2026-02-22T23:12:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:12:41 TEST: disable mirroring / delete non-primary image 2026-02-22T23:12:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image2=test2 2026-02-22T23:12:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image3=test3 2026-02-22T23:12:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image4=test4 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image5=test5 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test2 snapshot 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:12:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test2 2026-02-22T23:12:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:12:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:12:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test2 2026-02-22T23:12:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:12:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test2 snapshot 2026-02-22T23:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test2 snapshot 2026-02-22T23:12:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:12:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test2 2026-02-22T23:12:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test2': 2026-02-22T23:12:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:12:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:12:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:12:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 15d87777a804 2026-02-22T23:12:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.15d87777a804 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:12:41 2026 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:12:41 2026 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:12:41 2026 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 7b0c761c-e4d0-4373-b1e6-d4935673c092 2026-02-22T23:12:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:12:42.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T23:12:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:12:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:12:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:12:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:12:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:12:42.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:12:42.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 11111.2 bytes/sec: 43 MiB/s 2026-02-22T23:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test2 snap1 2026-02-22T23:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap1 2026-02-22T23:12:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:12:42.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test2 snap2 2026-02-22T23:12:42.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:42.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:42.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:42.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:12:42.999 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap2 2026-02-22T23:12:43.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:12:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' test2 = test4 ']' 2026-02-22T23:12:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' test2 = test5 ']' 2026-02-22T23:12:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T23:12:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:12:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:12:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:12:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:12:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:12:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:12:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:12:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:12:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test2 2026-02-22T23:12:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test2 2026-02-22T23:12:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 158 2026-02-22T23:12:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test2 present 2026-02-22T23:12:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=present 2026-02-22T23:12:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:12:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T23:12:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:45.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:45.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test2 2026-02-22T23:12:45.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T23:12:45.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:12:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id=10edeac40686 2026-02-22T23:12:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:12:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:12:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test2 present 10edeac40686 2026-02-22T23:12:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:12:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=10edeac40686 2026-02-22T23:12:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:12:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:12:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T23:12:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test2 2026-02-22T23:12:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T23:12:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:12:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=10edeac40686 2026-02-22T23:12:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 10edeac40686 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 10edeac40686 -o 10edeac40686 = 10edeac40686 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = present 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_present cluster1 mirror test2 snap2 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap2 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:12:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-02-22T23:12:46.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T23:12:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ continue 2026-02-22T23:12:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:12:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test2': 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 4 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 10edeac40686 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.10edeac40686 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, operations, non-primary 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: snap-trash 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:12:43 2026 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:12:43 2026 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:12:43 2026 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: protected: False 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 7b0c761c-e4d0-4373-b1e6-d4935673c092 2026-02-22T23:12:48.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: false 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test3 snapshot 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test3 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:12:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test3 2026-02-22T23:12:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:12:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:12:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test3 snapshot 2026-02-22T23:12:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test3 snapshot 2026-02-22T23:12:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:12:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test3 2026-02-22T23:12:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test3': 2026-02-22T23:12:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:12:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:12:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:12:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 15f0a3a7dd88 2026-02-22T23:12:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.15f0a3a7dd88 2026-02-22T23:12:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:12:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:12:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:12:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:12:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:12:48 2026 2026-02-22T23:12:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:12:48 2026 2026-02-22T23:12:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:12:48 2026 2026-02-22T23:12:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:12:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:12:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 0c86bd16-1b50-4875-a593-da3f0b429917 2026-02-22T23:12:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:12:49.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test3 100 2026-02-22T23:12:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:12:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:12:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:12:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:12:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:12:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:12:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 11111.2 bytes/sec: 43 MiB/s 2026-02-22T23:12:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test3 snap1 2026-02-22T23:12:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:12:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test3@snap1 2026-02-22T23:12:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:12:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test3 snap2 2026-02-22T23:12:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:12:50.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test3@snap2 2026-02-22T23:12:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:12:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' test3 = test4 ']' 2026-02-22T23:12:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' test3 = test5 ']' 2026-02-22T23:12:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test3 100 2026-02-22T23:12:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:12:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:12:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:12:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:12:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:12:51.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:12:51.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:12:51.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:12:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test3 2026-02-22T23:12:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test3 2026-02-22T23:12:52.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 162 2026-02-22T23:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test3 present 2026-02-22T23:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=present 2026-02-22T23:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T23:12:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test3 2026-02-22T23:12:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T23:12:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:12:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id=10f22d165dc9 2026-02-22T23:12:52.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:12:52.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test3 present 10f22d165dc9 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=10f22d165dc9 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test3 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T23:12:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:12:52.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=10f22d165dc9 2026-02-22T23:12:52.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 10f22d165dc9 2026-02-22T23:12:52.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 10f22d165dc9 -o 10f22d165dc9 = 10f22d165dc9 2026-02-22T23:12:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:12:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = present 2026-02-22T23:12:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_present cluster1 mirror test3 snap2 2026-02-22T23:12:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:12:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap2 2026-02-22T23:12:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:12:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test3@snap2 2026-02-22T23:12:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T23:12:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ continue 2026-02-22T23:12:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:12:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test3@snap2 2026-02-22T23:12:55.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test3': 2026-02-22T23:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 4 2026-02-22T23:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 10f22d165dc9 2026-02-22T23:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.10f22d165dc9 2026-02-22T23:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, operations, non-primary 2026-02-22T23:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: snap-trash 2026-02-22T23:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:12:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:12:51 2026 2026-02-22T23:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:12:51 2026 2026-02-22T23:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:12:51 2026 2026-02-22T23:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: protected: False 2026-02-22T23:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 0c86bd16-1b50-4875-a593-da3f0b429917 2026-02-22T23:12:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: false 2026-02-22T23:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T23:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test4 snapshot 2026-02-22T23:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test4 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:12:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:12:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:12:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test4 2026-02-22T23:12:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:12:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:12:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test4 snapshot 2026-02-22T23:12:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:55.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:55.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:12:55.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test4 snapshot 2026-02-22T23:12:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:12:56.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test4 2026-02-22T23:12:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test4': 2026-02-22T23:12:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:12:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 16073901bbf 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.16073901bbf 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:12:55 2026 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:12:55 2026 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:12:55 2026 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:12:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: c0df65ec-2c2a-4905-b5b9-5c7400fdf483 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test4 100 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:12:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:12:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:12:56.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:12:56.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 11111.2 bytes/sec: 43 MiB/s 2026-02-22T23:12:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test4 snap1 2026-02-22T23:12:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:12:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test4@snap1 2026-02-22T23:12:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:12:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test4 snap2 2026-02-22T23:12:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:12:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test4@snap2 2026-02-22T23:12:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:12:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' test4 = test4 ']' 2026-02-22T23:12:58.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ protect_snapshot cluster2 mirror test4 snap1 2026-02-22T23:12:58.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:58.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:58.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:58.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:12:58.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap protect mirror/test4@snap1 2026-02-22T23:12:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ protect_snapshot cluster2 mirror test4 snap2 2026-02-22T23:12:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:12:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap protect mirror/test4@snap2 2026-02-22T23:12:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test4 100 2026-02-22T23:12:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:12:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:12:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:12:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:12:58.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:12:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:12:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:12:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:12:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test4 2026-02-22T23:12:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:12:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test4 2026-02-22T23:12:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 166 2026-02-22T23:12:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test4 present 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=present 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test4 2026-02-22T23:12:59.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T23:12:59.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:12:59.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id=10fdbb0dbec9 2026-02-22T23:12:59.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:12:59.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:12:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test4 present 10fdbb0dbec9 2026-02-22T23:12:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:12:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=10fdbb0dbec9 2026-02-22T23:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T23:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test4 2026-02-22T23:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T23:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:12:59.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=10fdbb0dbec9 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 10fdbb0dbec9 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 10fdbb0dbec9 -o 10fdbb0dbec9 = 10fdbb0dbec9 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = present 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_present cluster1 mirror test4 snap2 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap2 2026-02-22T23:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:12:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:12:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test4@snap2 2026-02-22T23:13:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T23:13:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ continue 2026-02-22T23:13:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:13:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:13:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test4@snap2 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test4': 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 4 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 10fdbb0dbec9 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.10fdbb0dbec9 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, operations, non-primary 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: snap-trash 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:12:58 2026 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:12:58 2026 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:12:58 2026 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: protected: True 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: c0df65ec-2c2a-4905-b5b9-5c7400fdf483 2026-02-22T23:13:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: false 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test5 snapshot 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:13:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test5 2026-02-22T23:13:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:13:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:13:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test5 2026-02-22T23:13:02.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:13:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:13:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test5 snapshot 2026-02-22T23:13:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:13:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test5 snapshot 2026-02-22T23:13:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:13:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test5 2026-02-22T23:13:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test5': 2026-02-22T23:13:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:13:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:13:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:13:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 1625d780ca4 2026-02-22T23:13:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.1625d780ca4 2026-02-22T23:13:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:13:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:13:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:13:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:13:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:13:02 2026 2026-02-22T23:13:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:13:02 2026 2026-02-22T23:13:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:13:02 2026 2026-02-22T23:13:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:13:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:13:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 8250b256-22fd-4055-be58-cd68754b84d0 2026-02-22T23:13:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:13:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test5 100 2026-02-22T23:13:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:13:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:13:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:13:03.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:13:03.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:13:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:13:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:13:03.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test5 snap1 2026-02-22T23:13:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:13:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test5@snap1 2026-02-22T23:13:04.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:13:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test5 snap2 2026-02-22T23:13:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:13:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test5@snap2 2026-02-22T23:13:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:13:05.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' test5 = test4 ']' 2026-02-22T23:13:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' test5 = test5 ']' 2026-02-22T23:13:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ protect_snapshot cluster2 mirror test5 snap1 2026-02-22T23:13:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:13:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap protect mirror/test5@snap1 2026-02-22T23:13:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ protect_snapshot cluster2 mirror test5 snap2 2026-02-22T23:13:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:05.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:13:05.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap protect mirror/test5@snap2 2026-02-22T23:13:05.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test5 100 2026-02-22T23:13:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:13:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:13:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:13:05.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:13:05.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:13:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:13:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 480.774 bytes/sec: 1.9 MiB/s 2026-02-22T23:13:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:13:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test5 2026-02-22T23:13:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test5 2026-02-22T23:13:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 170 2026-02-22T23:13:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test5 present 2026-02-22T23:13:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=present 2026-02-22T23:13:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:13:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:13:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T23:13:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:13:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test5 2026-02-22T23:13:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T23:13:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:13:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id=10e6a9e7b98b 2026-02-22T23:13:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:13:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:13:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test5 present 10e6a9e7b98b 2026-02-22T23:13:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:13:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=10e6a9e7b98b 2026-02-22T23:13:06.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:13:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:13:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T23:13:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:13:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test5 2026-02-22T23:13:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T23:13:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:13:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=10e6a9e7b98b 2026-02-22T23:13:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 10e6a9e7b98b 2026-02-22T23:13:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 10e6a9e7b98b -o 10e6a9e7b98b = 10e6a9e7b98b 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = present 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_present cluster1 mirror test5 snap2 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap2 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:13:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test5@snap2 2026-02-22T23:13:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T23:13:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ continue 2026-02-22T23:13:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:13:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:13:09.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test5@snap2 2026-02-22T23:13:09.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test5': 2026-02-22T23:13:09.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:13:09.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:13:09.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 4 2026-02-22T23:13:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 10e6a9e7b98b 2026-02-22T23:13:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.10e6a9e7b98b 2026-02-22T23:13:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:13:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, operations, non-primary 2026-02-22T23:13:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: snap-trash 2026-02-22T23:13:09.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:13:09.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:13:05 2026 2026-02-22T23:13:09.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:13:05 2026 2026-02-22T23:13:09.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:13:05 2026 2026-02-22T23:13:09.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: protected: True 2026-02-22T23:13:09.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:13:09.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:13:09.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 8250b256-22fd-4055-be58-cd68754b84d0 2026-02-22T23:13:09.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: false 2026-02-22T23:13:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-22T23:13:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:09.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=image 2026-02-22T23:13:09.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-22T23:13:09.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd: mirroring is already configured for image mode 2026-02-22T23:13:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image4} 2026-02-22T23:13:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ disable_mirror cluster2 mirror test2 2026-02-22T23:13:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test2 2026-02-22T23:13:09.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring disabled 2026-02-22T23:13:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image4} 2026-02-22T23:13:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ disable_mirror cluster2 mirror test4 2026-02-22T23:13:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:13:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test4 2026-02-22T23:13:10.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring disabled 2026-02-22T23:13:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ unprotect_snapshot cluster2 mirror test5 snap1 2026-02-22T23:13:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:13:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test5@snap1 2026-02-22T23:13:10.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ unprotect_snapshot cluster2 mirror test5 snap2 2026-02-22T23:13:10.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:10.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:10.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:10.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:13:10.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test5@snap2 2026-02-22T23:13:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image3} ${image5} 2026-02-22T23:13:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_snapshot cluster2 mirror test3 snap1 2026-02-22T23:13:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:13:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:13:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rm mirror/test3@snap1 2026-02-22T23:13:11.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing snap: 100% complete...done. 2026-02-22T23:13:11.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_snapshot cluster2 mirror test3 snap2 2026-02-22T23:13:11.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:11.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:11.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:13:11.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:13:11.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rm mirror/test3@snap2 2026-02-22T23:13:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing snap: 100% complete...done. 2026-02-22T23:13:12.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test3 2026-02-22T23:13:12.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:12.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:12.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:13:12.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:12.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:13:12.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test3 2026-02-22T23:13:12.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:12.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:12.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:13:12.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test3 2026-02-22T23:13:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test3 2026-02-22T23:13:13.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:13:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image3} ${image5} 2026-02-22T23:13:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_snapshot cluster2 mirror test5 snap1 2026-02-22T23:13:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:13:13.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rm mirror/test5@snap1 2026-02-22T23:13:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing snap: 100% complete...done. 2026-02-22T23:13:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_snapshot cluster2 mirror test5 snap2 2026-02-22T23:13:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:13:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rm mirror/test5@snap2 2026-02-22T23:13:15.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing snap: 100% complete...done. 2026-02-22T23:13:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test5 2026-02-22T23:13:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:13:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test5 2026-02-22T23:13:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:15.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test5 2026-02-22T23:13:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test5 2026-02-22T23:13:16.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:13:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T23:13:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test2 deleted 2026-02-22T23:13:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:13:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:13:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:13:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T23:13:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:13:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test2 2026-02-22T23:13:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T23:13:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:13:16.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-22T23:13:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id= 2026-02-22T23:13:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:13:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test2 deleted '' 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test2 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T23:13:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:13:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test3 deleted 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T23:13:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:13:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test3 2026-02-22T23:13:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T23:13:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:13:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test3: (2) No such file or directory 2026-02-22T23:13:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id= 2026-02-22T23:13:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:13:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:13:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test3 deleted '' 2026-02-22T23:13:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test3 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test3 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T23:13:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:13:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test3: (2) No such file or directory 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test4 deleted 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:13:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test4 2026-02-22T23:13:16.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T23:13:16.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:13:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T23:13:16.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id= 2026-02-22T23:13:16.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:13:16.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:13:16.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test4 deleted '' 2026-02-22T23:13:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:13:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:13:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:13:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:13:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:13:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T23:13:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:13:16.860 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test4 2026-02-22T23:13:16.860 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T23:13:16.860 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:13:16.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T23:13:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:13:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test5 deleted 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T23:13:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:13:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test5 2026-02-22T23:13:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T23:13:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:13:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test5: (2) No such file or directory 2026-02-22T23:13:16.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id= 2026-02-22T23:13:16.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:13:16.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:13:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test5 deleted '' 2026-02-22T23:13:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test5 2026-02-22T23:13:17.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:13:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:13:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:13:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:13:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T23:13:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:13:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test5 2026-02-22T23:13:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T23:13:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:13:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test5: (2) No such file or directory 2026-02-22T23:13:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:13:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:13:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:13:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: snapshot rename' 2026-02-22T23:13:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:13:17.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:13:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:13:17 TEST: snapshot rename 2026-02-22T23:13:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:13:17 TEST: snapshot rename 2026-02-22T23:13:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ snap_name=snap_rename 2026-02-22T23:13:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:13:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test2 2026-02-22T23:13:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:13:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test2 snapshot 2026-02-22T23:13:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:13:17.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test2 2026-02-22T23:13:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test2': 2026-02-22T23:13:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:13:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:13:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 3 2026-02-22T23:13:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 15d87777a804 2026-02-22T23:13:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.15d87777a804 2026-02-22T23:13:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:13:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:13:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:13:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:13:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:12:41 2026 2026-02-22T23:13:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:12:41 2026 2026-02-22T23:13:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:12:41 2026 2026-02-22T23:13:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:13:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:13:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 2bb541f4-ee8e-4f4b-a03b-77d7e2641c93 2026-02-22T23:13:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:13:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test2 snap_rename_0 2026-02-22T23:13:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_0 2026-02-22T23:13:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap_rename_0 2026-02-22T23:13:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:13:18.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 1 20 2026-02-22T23:13:18.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 1 - 1 2026-02-22T23:13:18.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_0 snap_rename_1 2026-02-22T23:13:18.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_0 2026-02-22T23:13:18.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_1 2026-02-22T23:13:18.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_0 mirror/test2@snap_rename_1 2026-02-22T23:13:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 2 - 1 2026-02-22T23:13:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_1 snap_rename_2 2026-02-22T23:13:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_1 2026-02-22T23:13:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_2 2026-02-22T23:13:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_1 mirror/test2@snap_rename_2 2026-02-22T23:13:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 3 - 1 2026-02-22T23:13:18.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_2 snap_rename_3 2026-02-22T23:13:18.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_2 2026-02-22T23:13:18.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_3 2026-02-22T23:13:18.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_2 mirror/test2@snap_rename_3 2026-02-22T23:13:18.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 4 - 1 2026-02-22T23:13:18.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_3 snap_rename_4 2026-02-22T23:13:18.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_3 2026-02-22T23:13:18.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_4 2026-02-22T23:13:18.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_3 mirror/test2@snap_rename_4 2026-02-22T23:13:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 5 - 1 2026-02-22T23:13:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_4 snap_rename_5 2026-02-22T23:13:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_4 2026-02-22T23:13:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_5 2026-02-22T23:13:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_4 mirror/test2@snap_rename_5 2026-02-22T23:13:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.561 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 6 - 1 2026-02-22T23:13:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_5 snap_rename_6 2026-02-22T23:13:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_5 2026-02-22T23:13:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_6 2026-02-22T23:13:18.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_5 mirror/test2@snap_rename_6 2026-02-22T23:13:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 7 - 1 2026-02-22T23:13:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_6 snap_rename_7 2026-02-22T23:13:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_6 2026-02-22T23:13:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_7 2026-02-22T23:13:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_6 mirror/test2@snap_rename_7 2026-02-22T23:13:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 8 - 1 2026-02-22T23:13:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_7 snap_rename_8 2026-02-22T23:13:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_7 2026-02-22T23:13:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_8 2026-02-22T23:13:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_7 mirror/test2@snap_rename_8 2026-02-22T23:13:18.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 9 - 1 2026-02-22T23:13:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_8 snap_rename_9 2026-02-22T23:13:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_8 2026-02-22T23:13:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_9 2026-02-22T23:13:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_8 mirror/test2@snap_rename_9 2026-02-22T23:13:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 10 - 1 2026-02-22T23:13:18.677 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_9 snap_rename_10 2026-02-22T23:13:18.677 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_9 2026-02-22T23:13:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_10 2026-02-22T23:13:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_9 mirror/test2@snap_rename_10 2026-02-22T23:13:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 11 - 1 2026-02-22T23:13:18.707 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_10 snap_rename_11 2026-02-22T23:13:18.707 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.707 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_10 2026-02-22T23:13:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_11 2026-02-22T23:13:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_10 mirror/test2@snap_rename_11 2026-02-22T23:13:18.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.735 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 12 - 1 2026-02-22T23:13:18.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_11 snap_rename_12 2026-02-22T23:13:18.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_11 2026-02-22T23:13:18.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_12 2026-02-22T23:13:18.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_11 mirror/test2@snap_rename_12 2026-02-22T23:13:18.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 13 - 1 2026-02-22T23:13:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_12 snap_rename_13 2026-02-22T23:13:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_12 2026-02-22T23:13:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_13 2026-02-22T23:13:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_12 mirror/test2@snap_rename_13 2026-02-22T23:13:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 14 - 1 2026-02-22T23:13:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_13 snap_rename_14 2026-02-22T23:13:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_13 2026-02-22T23:13:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_14 2026-02-22T23:13:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_13 mirror/test2@snap_rename_14 2026-02-22T23:13:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 15 - 1 2026-02-22T23:13:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_14 snap_rename_15 2026-02-22T23:13:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_14 2026-02-22T23:13:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_15 2026-02-22T23:13:19.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_14 mirror/test2@snap_rename_15 2026-02-22T23:13:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 16 - 1 2026-02-22T23:13:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_15 snap_rename_16 2026-02-22T23:13:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_15 2026-02-22T23:13:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_16 2026-02-22T23:13:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_15 mirror/test2@snap_rename_16 2026-02-22T23:13:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 17 - 1 2026-02-22T23:13:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_16 snap_rename_17 2026-02-22T23:13:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_16 2026-02-22T23:13:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_17 2026-02-22T23:13:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_16 mirror/test2@snap_rename_17 2026-02-22T23:13:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:19.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 18 - 1 2026-02-22T23:13:19.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_17 snap_rename_18 2026-02-22T23:13:19.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:19.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:19.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:19.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_17 2026-02-22T23:13:19.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_18 2026-02-22T23:13:19.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_17 mirror/test2@snap_rename_18 2026-02-22T23:13:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 19 - 1 2026-02-22T23:13:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_18 snap_rename_19 2026-02-22T23:13:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_18 2026-02-22T23:13:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_19 2026-02-22T23:13:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_18 mirror/test2@snap_rename_19 2026-02-22T23:13:19.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in `seq 1 20` 2026-02-22T23:13:19.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ expr 20 - 1 2026-02-22T23:13:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_19 snap_rename_20 2026-02-22T23:13:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap_rename_19 2026-02-22T23:13:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local new_snap=snap_rename_20 2026-02-22T23:13:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_19 mirror/test2@snap_rename_20 2026-02-22T23:13:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:13:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test2 2026-02-22T23:13:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test2 2026-02-22T23:13:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 173 2026-02-22T23:13:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_present cluster1 mirror test2 snap_rename_20 2026-02-22T23:13:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap_rename_20 2026-02-22T23:13:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:13:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:13:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:21.029 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-22T23:13:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T23:13:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ continue 2026-02-22T23:13:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:13:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:13:23.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-22T23:13:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test2': 2026-02-22T23:13:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:13:23.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:13:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 5 2026-02-22T23:13:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 10ed9c31efc8 2026-02-22T23:13:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.10ed9c31efc8 2026-02-22T23:13:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:13:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, operations, non-primary 2026-02-22T23:13:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: snap-trash 2026-02-22T23:13:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:13:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:13:18 2026 2026-02-22T23:13:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:13:18 2026 2026-02-22T23:13:23.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:13:18 2026 2026-02-22T23:13:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: protected: False 2026-02-22T23:13:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:13:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:13:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 2bb541f4-ee8e-4f4b-a03b-77d7e2641c93 2026-02-22T23:13:23.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: false 2026-02-22T23:13:23.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:23.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ unprotect_snapshot cluster2 mirror test4 snap1 2026-02-22T23:13:23.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:23.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:23.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:13:23.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:13:23.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test4@snap1 2026-02-22T23:13:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ unprotect_snapshot cluster2 mirror test4 snap2 2026-02-22T23:13:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:13:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap2 2026-02-22T23:13:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test4@snap2 2026-02-22T23:13:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image4} 2026-02-22T23:13:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test2 2026-02-22T23:13:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:13:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test2 2026-02-22T23:13:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test2 2026-02-22T23:13:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test2 2026-02-22T23:13:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing all snapshots: 33% complete... Removing all snapshots: 66% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T23:13:26.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test2 2026-02-22T23:13:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:13:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for i in ${image2} ${image4} 2026-02-22T23:13:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test4 2026-02-22T23:13:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:13:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:13:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test4 2026-02-22T23:13:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test4 2026-02-22T23:13:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test4 2026-02-22T23:13:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T23:13:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test4 2026-02-22T23:13:29.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:13:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z '' ']' 2026-02-22T23:13:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: demote image while daemon is offline' 2026-02-22T23:13:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:13:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:13:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:13:29 TEST: demote image while daemon is offline 2026-02-22T23:13:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:13:29 TEST: demote image while daemon is offline 2026-02-22T23:13:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image=test_demote_image 2026-02-22T23:13:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test_demote_image snapshot 2026-02-22T23:13:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:29.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:13:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:13:29.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test_demote_image 2026-02-22T23:13:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:29.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:29.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:13:29.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:13:29.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:13:29.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test_demote_image 2026-02-22T23:13:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:13:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:13:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test_demote_image snapshot 2026-02-22T23:13:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:13:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_demote_image snapshot 2026-02-22T23:13:30.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:13:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test_demote_image 2026-02-22T23:13:30.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_demote_image': 2026-02-22T23:13:30.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:13:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:13:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:13:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 16c3dd4debb 2026-02-22T23:13:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.16c3dd4debb 2026-02-22T23:13:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:13:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:13:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:13:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:13:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:13:29 2026 2026-02-22T23:13:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:13:29 2026 2026-02-22T23:13:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:13:29 2026 2026-02-22T23:13:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:13:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:13:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: da88721c-905f-46a3-8189-207b76ec3b80 2026-02-22T23:13:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:13:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-22T23:13:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:13:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:13:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:13:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:13:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:13:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:13:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 11111.2 bytes/sec: 43 MiB/s 2026-02-22T23:13:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_demote_image 2026-02-22T23:13:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-22T23:13:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:13:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:13:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:30.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:31.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-22T23:13:31.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:31.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:31.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:31.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:13:31.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:13:31.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:13:31.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:31.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:13:31.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:13:31.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:31.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:13:31.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:13:31.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:13:31.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:13:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:13:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:13:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:13:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:13:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:13:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:13:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:13:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:13:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:13:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:13:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:13:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:13:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:13:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:13:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:13:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:13:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:31.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:31.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:13:31.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:13:31.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:13:31.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:13:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:13:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:13:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:13:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:13:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:13:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:13:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:13:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:13:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:13:31.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:13:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:2 2026-02-22T23:13:31.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:13:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:2 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:2 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T23:13:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:13:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:13:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T23:13:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:13:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:13:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:13:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:31.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:13:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:13:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:13:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:3 2026-02-22T23:13:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T23:13:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:3 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:3 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:13:31.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:13:31.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T23:13:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:13:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:13:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:13:31.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:13:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:13:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:13:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:13:32.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:13:32.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:32.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:32.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:32.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:13:32.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:32.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:32.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:13:32.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:13:32.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:13:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:32.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:13:32.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:13:32.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:13:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:13:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:32.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:32.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:13:32.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:13:32.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:13:32.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:13:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:13:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_demote_image 2026-02-22T23:13:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_demote_image started 2026-02-22T23:13:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:13:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:13:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:32.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_demote_image started 2026-02-22T23:13:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:13:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:13:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:13:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:13:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:13:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:13:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:13:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:13:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:13:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:13:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:13:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:13:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:13:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:13:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:13:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:13:33.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:13:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:13:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:13:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:13:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:33.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:13:33.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:13:33.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:13:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:13:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:13:33.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:13:33.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:13:33.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:13:33.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:33.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:13:33.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:13:33.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:13:33.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:13:33.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:13:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:13:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:13:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T23:13:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:13:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:33.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:13:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:13:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test_demote_image 2026-02-22T23:13:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_demote_image 2026-02-22T23:13:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 175 2026-02-22T23:13:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T23:13:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:13:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T23:13:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-22T23:13:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:13:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:13:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-22T23:13:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-22T23:13:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:13:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:13:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:13:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:13:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:13:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:13:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=175 2026-02-22T23:13:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:13:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:13:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:13:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:13:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:13:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:13:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=174 2026-02-22T23:13:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 175 = 174 2026-02-22T23:13:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:13:34.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:13:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:13:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:13:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:13:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=174 2026-02-22T23:13:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 175 = 174 2026-02-22T23:13:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:13:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T23:13:35.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:35.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:35.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:35.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:35.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:35.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:13:35.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:13:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:13:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=175 2026-02-22T23:13:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 175 = 175 2026-02-22T23:13:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+replaying 2026-02-22T23:13:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:35.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:35.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:13:35.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:13:35.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:13:35.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:35.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+replaying '' '' 2026-02-22T23:13:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:13:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:13:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:13:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:13:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:13:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T23:13:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:13:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: da88721c-905f-46a3-8189-207b76ec3b80 2026-02-22T23:13:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:13:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802010,"replay_state":"idle"} 2026-02-22T23:13:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:13:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:32 2026-02-22T23:13:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:13:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:13:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:13:36.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:13:36.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:35' 2026-02-22T23:13:36.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:13:36.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:13:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:13:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802010,"replay_state":"idle"} 2026-02-22T23:13:36.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:13:36.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:13:36.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:13:36.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:13:36.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:13:36.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:13:36.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:13:36.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:13:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:32' 2026-02-22T23:13:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+replaying '2026-02-22 23:13:32' 2026-02-22T23:13:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:13:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:13:32' 2026-02-22T23:13:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:13:36.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:13:36.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:13:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK243224326mirror.0trial08219.2.3-689-gf90f4eaetrueOK43284333mirror.1trial08219.2.3-689-gf90f4eaefalseOK43354338mirror.2trial08219.2.3-689-gf90f4eaefalseOK43434349mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}43224326mirror.0trial0822026-02-22 23:13:23cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:13:35test_demote_imageda88721c-905f-46a3-8189-207b76ec3b80up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802010,"replay_state":"idle"}43284333mirror.1trial0822026-02-22 23:13:32cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:13:35' 2026-02-22T23:13:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:13:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:13:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:32' 2026-02-22T23:13:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:13:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:13:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:13:36.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:13:36.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:13:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:36.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+stopped 2026-02-22T23:13:36.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:36.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:36.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:36.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:13:36.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:13:36.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:13:36.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:36.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:37.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+stopped '' '' 2026-02-22T23:13:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:13:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:13:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:13:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:13:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:13:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T23:13:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: da88721c-905f-46a3-8189-207b76ec3b80 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:35 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802010,"replay_state":"idle"} 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:32 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 174 .mirror.primary.da88721c-905f-46a3-8189-207b76ec3b80.b19f5ccb-3a88-4a76-af10-46f33b068b2d (peer_uuids:[]) 2026-02-22T23:13:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 175 .mirror.primary.da88721c-905f-46a3-8189-207b76ec3b80.55834889-647c-4d23-b07d-6f3fcd2ac0cd (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:13:37.677 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:13:37.677 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:13:37.677 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:13:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:13:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:13:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:13:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:13:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:13:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:13:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:13:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:13:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:13:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:35' 2026-02-22T23:13:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+stopped '2026-02-22 23:13:35' 2026-02-22T23:13:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:13:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:13:35' 2026-02-22T23:13:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:13:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:13:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:13:37.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK244464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:13:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}2026-02-22 23:13:23test_demote_imageda88721c-905f-46a3-8189-207b76ec3b80up+stoppedlocal image is primary44534458mirror.1trial0822026-02-22 23:13:35cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802010,"replay_state":"idle"}2026-02-22 23:13:32' 2026-02-22T23:13:37.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:13:37.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:13:37.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:35' 2026-02-22T23:13:37.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:13:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:13:37.716 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:13:37.716 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:13:37.716 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:13:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirrors cluster1 2026-02-22T23:13:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:13:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:13:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T23:13:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:13:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:13:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:13:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:0 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:13:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:13:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:13:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:13:37.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:13:37.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:13:37.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=50406 2026-02-22T23:13:37.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 50406 ']' 2026-02-22T23:13:37.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 50406 2026-02-22T23:13:37.732 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:37.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:13:37.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:37.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50406 '$2 == pid {print; exit 1}' 2026-02-22T23:13:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 50406 0.2 0.0 779516 71248 ? Rsl 22:57 0:02 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:13:37.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:37.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:38.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50406 '$2 == pid {print; exit 1}' 2026-02-22T23:13:38.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:13:38.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:38.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50406 '$2 == pid {print; exit 1}' 2026-02-22T23:13:38.772 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:13:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:13:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:13:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:13:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:13:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:13:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:13:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:13:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:13:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:13:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:13:38.784 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:13:38.784 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:13:38.785 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T23:13:38.785 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:13:38.785 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:38.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:13:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:13:38.798 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:13:38.799 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:0 2026-02-22T23:13:38.799 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:13:38.799 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:38.799 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:38.800 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:13:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:13:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:13:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T23:13:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:13:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:13:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:1 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:13:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:13:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:13:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:13:38.816 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:13:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:13:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:13:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=50445 2026-02-22T23:13:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 50445 ']' 2026-02-22T23:13:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 50445 2026-02-22T23:13:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:13:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50445 '$2 == pid {print; exit 1}' 2026-02-22T23:13:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 50445 0.1 0.0 531588 62760 ? Ssl 22:57 0:01 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:13:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:38.840 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:39.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:39.841 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50445 '$2 == pid {print; exit 1}' 2026-02-22T23:13:39.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:13:39.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:39.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50445 '$2 == pid {print; exit 1}' 2026-02-22T23:13:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:13:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:13:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:13:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:13:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:13:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:13:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:13:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:13:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:13:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:13:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:13:39.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:13:39.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:13:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T23:13:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:13:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:13:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:13:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:13:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:13:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:13:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:13:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:13:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:13:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:13:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:13:39.889 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:13:39.889 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:13:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:1 2026-02-22T23:13:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:13:39.890 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:13:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:13:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:13:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T23:13:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:13:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:13:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:2 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:13:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:13:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:13:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:13:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=50510 2026-02-22T23:13:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 50510 ']' 2026-02-22T23:13:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 50510 2026-02-22T23:13:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:13:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50510 '$2 == pid {print; exit 1}' 2026-02-22T23:13:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 50510 0.0 0.0 736484 61520 ? Ssl 22:57 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:13:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50510 '$2 == pid {print; exit 1}' 2026-02-22T23:13:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:13:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50510 '$2 == pid {print; exit 1}' 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:13:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:13:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:13:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:13:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T23:13:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:13:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:13:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:13:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:13:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:13:40.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:13:40.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:13:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:2 2026-02-22T23:13:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:13:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:13:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:13:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:13:40.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:13:40.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:13:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T23:13:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:13:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:13:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:3 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:13:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:13:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:13:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:13:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=50572 2026-02-22T23:13:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 50572 ']' 2026-02-22T23:13:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 50572 2026-02-22T23:13:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:13:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50572 '$2 == pid {print; exit 1}' 2026-02-22T23:13:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 50572 0.0 0.0 736492 60804 ? Ssl 22:57 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:13:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:13:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50572 '$2 == pid {print; exit 1}' 2026-02-22T23:13:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:13:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:13:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=50572 '$2 == pid {print; exit 1}' 2026-02-22T23:13:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:13:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:13:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:13:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:13:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:13:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:13:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:13:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:13:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:13:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:13:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:13:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:13:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:13:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:13:42.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:13:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:13:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:13:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:13:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:13:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:13:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:13:42.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:13:42.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:13:42.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:3 2026-02-22T23:13:42.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:13:42.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:42.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:13:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:13:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:13:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:13:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-22T23:13:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:13:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:13:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:13:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:13:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:13:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:13:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:13:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test_demote_image 2026-02-22T23:13:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_demote_image 2026-02-22T23:13:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:13:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirrors cluster1 2026-02-22T23:13:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:13:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:0 2026-02-22T23:13:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:13:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:13:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:13:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T23:13:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:13:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:13:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:13:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T23:13:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T23:13:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:13:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T23:13:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:13:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:13:44.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:44.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:1 2026-02-22T23:13:44.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:13:44.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:13:44.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:13:44.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T23:13:44.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:13:44.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:13:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:13:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T23:13:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T23:13:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:13:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T23:13:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:13:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:13:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:2 2026-02-22T23:13:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T23:13:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:13:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:13:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:3 2026-02-22T23:13:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:13:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:13:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:13:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T23:13:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:13:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:13:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:13:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T23:13:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T23:13:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:13:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:13:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T23:13:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:13:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:13:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:13:44.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T23:13:44.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:13:44.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:44.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:44.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:44.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T23:13:44.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-22T23:13:44.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:13:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:13:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-22T23:13:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-22T23:13:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:13:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:13:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:13:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:13:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:13:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:13:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=176 2026-02-22T23:13:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:13:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:13:44.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:13:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:13:44.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:13:44.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:13:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=175 2026-02-22T23:13:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 176 = 175 2026-02-22T23:13:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:13:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:13:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:13:44.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:13:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:13:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=175 2026-02-22T23:13:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 176 = 175 2026-02-22T23:13:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:13:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T23:13:45.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:45.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:45.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:45.625 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:45.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:45.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:13:45.626 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:13:45.656 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:13:45.658 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=175 2026-02-22T23:13:45.658 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 176 = 175 2026-02-22T23:13:45.658 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:13:45.658 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1.6 2026-02-22T23:13:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:13:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:13:47.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:13:47.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=175 2026-02-22T23:13:47.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 176 = 175 2026-02-22T23:13:47.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:13:47.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:13:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:13:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:13:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:13:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:13:49.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=176 2026-02-22T23:13:49.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 176 = 176 2026-02-22T23:13:49.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:49.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+unknown 2026-02-22T23:13:49.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:49.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:49.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:13:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:13:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:13:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+unknown '' '' 2026-02-22T23:13:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:13:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:13:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:13:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:13:50.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:13:50.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T23:13:50.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: da88721c-905f-46a3-8189-207b76ec3b80 2026-02-22T23:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:45 2026-02-22T23:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:50' 2026-02-22T23:13:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:13:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:13:50.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:13:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:13:50.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:13:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:13:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:13:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:13:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:13:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:13:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:13:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:13:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:45' 2026-02-22T23:13:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+unknown '2026-02-22 23:13:45' 2026-02-22T23:13:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:13:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:13:45' 2026-02-22T23:13:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:13:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:13:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:13:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING1144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:13:45cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}2026-02-22 23:13:48test_demote_imageda88721c-905f-46a3-8189-207b76ec3b80up+unknownremote image is not primary44534458mirror.1trial0822026-02-22 23:13:45cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image demoted2026-02-22 23:13:50' 2026-02-22T23:13:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:13:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:13:50.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:45' 2026-02-22T23:13:50.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:13:50.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:13:50.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:13:50.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:13:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:13:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+unknown 2026-02-22T23:13:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:13:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:13:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:13:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:50.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-02-22T23:13:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:13:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:13:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:13:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:13:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:13:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T23:13:51.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:13:51.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: da88721c-905f-46a3-8189-207b76ec3b80 2026-02-22T23:13:51.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:13:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:13:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:13:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:50 2026-02-22T23:13:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:13:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:13:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:13:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:13:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:50' 2026-02-22T23:13:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:13:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:13:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:13:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image demoted 2026-02-22T23:13:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:13:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:13:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:13:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:13:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:13:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:13:51.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:13:51.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:13:51.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:50' 2026-02-22T23:13:51.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+unknown '2026-02-22 23:13:50' 2026-02-22T23:13:51.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:51.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:51.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:51.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:13:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:13:50' 2026-02-22T23:13:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:13:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:13:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:13:51.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING1156305635mirror.0trial08219.2.3-689-gf90f4eaetrueOK56375642mirror.1trial08219.2.3-689-gf90f4eaefalseOK56445648mirror.2trial08219.2.3-689-gf90f4eaefalseOK56505654mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}56305635mirror.0trial0822026-02-22 23:13:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:13:50test_demote_imageda88721c-905f-46a3-8189-207b76ec3b80up+unknownremote image demoted56375642mirror.1trial0822026-02-22 23:13:50cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:13:50' 2026-02-22T23:13:51.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:13:51.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:13:51.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:50' 2026-02-22T23:13:51.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:13:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:13:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:13:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:13:51.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:13:51.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:51.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:51.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test_demote_image 2026-02-22T23:13:51.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:51.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:51.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:13:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_demote_image.export 2026-02-22T23:13:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_demote_image.export 2026-02-22T23:13:51.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:13:51.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.export 2026-02-22T23:13:51.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_demote_image.export 2026-02-22T23:13:51.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_demote_image.export 2026-02-22T23:13:51.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:13:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.export 2026-02-22T23:13:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.export 2026-02-22T23:13:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test_demote_image /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.export 2026-02-22T23:13:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:13:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.export 2026-02-22T23:13:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:13:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.export 2026-02-22T23:13:51.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.export 2026-02-22T23:13:51.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test_demote_image 2026-02-22T23:13:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:13:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_demote_image 2026-02-22T23:13:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:13:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test_demote_image 2026-02-22T23:13:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T23:13:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:13:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:13:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:13:53.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T23:13:53.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:53.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:53.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:53.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:13:53.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:13:53.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:13:53.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:53.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:13:53.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:13:53.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:53.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:13:53.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:13:53.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:13:53.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:53.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:13:53.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:13:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:13:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:13:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:13:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:13:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:13:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:13:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:13:53.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:53.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:53.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:53.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:13:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:13:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:13:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:13:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:13:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:13:53.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:53.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:13:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:13:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:13:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:13:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:13:53.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:13:53.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:53.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:53.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:53.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:13:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:13:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:13:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:13:53.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:13:53.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:53.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:13:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T23:13:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:13:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:13:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:13:55.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:13:55.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:13:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:13:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:13:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:13:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:13:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:13:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:55.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:13:55.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:13:55.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:13:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:13:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:13:55.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:13:55.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:13:55.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:13:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:55.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:13:55.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:13:55.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:13:55.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:13:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:13:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:13:59.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T23:13:59.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:13:59.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:59.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:59.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:13:59.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:13:59.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:13:59.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:59.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:13:59.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:59.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:13:59.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:13:59.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:13:59.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:59.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:13:59.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:13:59.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:13:59.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:13:59.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:13:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:59.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:59.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:13:59.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:13:59.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:59.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:59.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:13:59.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:13:59.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:13:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:13:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:13:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:13:59.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:13:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:13:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:13:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:13:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:13:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:13:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:13:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:13:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:13:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:13:59.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:59.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:59.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:13:59.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:13:59.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:13:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:13:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:13:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:13:59.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:13:59.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:13:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:13:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:13:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+stopped 2026-02-22T23:13:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:13:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:13:59.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:13:59.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:13:59.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:13:59.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:13:59.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:13:59.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:00.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+stopped '' '' 2026-02-22T23:14:00.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:00.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:00.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:14:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T23:14:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:14:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: da88721c-905f-46a3-8189-207b76ec3b80 2026-02-22T23:14:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:59 2026-02-22T23:14:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:14:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802032,"replay_state":"idle"} 2026-02-22T23:14:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:55 2026-02-22T23:14:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:14:00.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 181 .mirror.primary.da88721c-905f-46a3-8189-207b76ec3b80.01fcc709-73d0-4d79-b549-611b46dc7d75 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:14:00.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:14:00.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:14:00.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:14:00.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:14:00.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:00.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:14:00.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:00.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:14:00.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:00.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:00.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:00.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:59' 2026-02-22T23:14:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+stopped '2026-02-22 23:13:59' 2026-02-22T23:14:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:14:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:13:59' 2026-02-22T23:14:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:00.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK256305635mirror.0trial08219.2.3-689-gf90f4eaetrueOK56375642mirror.1trial08219.2.3-689-gf90f4eaefalseOK56445648mirror.2trial08219.2.3-689-gf90f4eaefalseOK56505654mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}56305635mirror.0trial0822026-02-22 23:13:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:13:55test_demote_imageda88721c-905f-46a3-8189-207b76ec3b80up+stoppedlocal image is primary56375642mirror.1trial0822026-02-22 23:13:59cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802032,"replay_state":"idle"}2026-02-22 23:13:55' 2026-02-22T23:14:00.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:00.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:14:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:59' 2026-02-22T23:14:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:14:00.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:14:00.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:14:00.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:14:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:14:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+replaying 2026-02-22T23:14:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:14:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+replaying '' '' 2026-02-22T23:14:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:14:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T23:14:01.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:14:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: da88721c-905f-46a3-8189-207b76ec3b80 2026-02-22T23:14:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802032,"replay_state":"idle"} 2026-02-22T23:14:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:55 2026-02-22T23:14:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:14:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:01.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:01.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:13:59' 2026-02-22T23:14:01.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:14:01.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:14:01.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:14:01.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802032,"replay_state":"idle"} 2026-02-22T23:14:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:14:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:01.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:14:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:01.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:01.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:55' 2026-02-22T23:14:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+replaying '2026-02-22 23:13:55' 2026-02-22T23:14:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:14:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:13:55' 2026-02-22T23:14:01.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:01.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK244464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:14:00cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}2026-02-22 23:13:48test_demote_imageda88721c-905f-46a3-8189-207b76ec3b80up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802032,"replay_state":"idle"}44534458mirror.1trial0822026-02-22 23:13:55cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:13:59' 2026-02-22T23:14:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:14:01.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:13:55' 2026-02-22T23:14:01.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:14:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:14:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:14:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:14:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:14:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster1 mirror test_demote_image 2026-02-22T23:14:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:14:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster1 mirror test_demote_image 2026-02-22T23:14:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:14:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_demote_image 2026-02-22T23:14:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 rm mirror/test_demote_image 2026-02-22T23:14:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:14:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:14:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: request image resync when remote is not primary' 2026-02-22T23:14:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:14:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:14:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:14:02 TEST: request image resync when remote is not primary 2026-02-22T23:14:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:14:02 TEST: request image resync when remote is not primary 2026-02-22T23:14:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_resync_image=test_resync_image 2026-02-22T23:14:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-22T23:14:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test_resync_image 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:14:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test_resync_image 2026-02-22T23:14:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:14:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:14:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-22T23:14:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:14:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_resync_image snapshot 2026-02-22T23:14:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:14:03.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test_resync_image 2026-02-22T23:14:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_resync_image': 2026-02-22T23:14:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:14:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:14:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:14:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 1703693521f 2026-02-22T23:14:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.1703693521f 2026-02-22T23:14:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:14:03.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:14:03.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:14:03.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:14:03.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:14:02 2026 2026-02-22T23:14:03.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:14:02 2026 2026-02-22T23:14:03.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:14:02 2026 2026-02-22T23:14:03.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:14:03.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:14:03.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 2d0ac1d1-d0c0-4648-bbba-c710a4ad1078 2026-02-22T23:14:03.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:14:03.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-22T23:14:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:14:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:14:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:14:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:14:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:14:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:14:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 11111.2 bytes/sec: 43 MiB/s 2026-02-22T23:14:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_resync_image 2026-02-22T23:14:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-22T23:14:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:14:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:14:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:04.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-22T23:14:04.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:14:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:14:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:14:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:04.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:14:04.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:14:04.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:04.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:04.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:14:04.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:14:04.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:04.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:14:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:14:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:14:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:14:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:04.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:04.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:04.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:14:04.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:14:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:2 2026-02-22T23:14:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:14:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:14:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:2 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:2 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T23:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:14:04.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T23:14:04.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:14:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:14:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:14:04.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:04.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:04.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:04.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:04.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:04.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:04.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:3 2026-02-22T23:14:04.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:04.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:14:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:14:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:04.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:14:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:14:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:14:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:14:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T23:14:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:3 2026-02-22T23:14:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:3 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:14:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:14:04.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T23:14:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:14:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:14:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:14:04.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:04.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:04.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:04.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:14:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:14:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:14:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:14:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:14:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:14:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:14:05.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:05.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:05.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:05.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:05.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:05.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:05.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:14:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:14:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:14:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:14:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:14:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:05.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:05.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:05.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:14:05.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:14:05.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:14:05.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:14:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:14:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_resync_image 2026-02-22T23:14:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_resync_image started 2026-02-22T23:14:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:05.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:14:05.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:14:05.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:05.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_resync_image started 2026-02-22T23:14:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:14:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:14:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:14:06.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:14:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:14:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:14:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:14:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:14:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:14:06.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:14:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:14:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:14:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:14:06.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:06.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:06.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:06.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:06.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:06.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:06.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:14:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:14:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:14:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:14:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:06.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:14:06.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:14:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:14:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:14:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:14:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:14:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:14:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_resync_image 2026-02-22T23:14:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:14:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:14:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:14:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test_resync_image 2026-02-22T23:14:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_resync_image 2026-02-22T23:14:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 180 2026-02-22T23:14:08.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_resync_image 2026-02-22T23:14:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:14:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_resync_image.status 2026-02-22T23:14:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_resync_image.status 2026-02-22T23:14:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:14:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.status 2026-02-22T23:14:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_resync_image.status 2026-02-22T23:14:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_resync_image.status 2026-02-22T23:14:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:14:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_resync_image /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.status 2026-02-22T23:14:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.status 2026-02-22T23:14:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_resync_image --format xml 2026-02-22T23:14:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:14:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:14:08.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=180 2026-02-22T23:14:08.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:14:08.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:14:08.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:14:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:08.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:08.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-22T23:14:08.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:14:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:14:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=178 2026-02-22T23:14:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 180 = 178 2026-02-22T23:14:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:14:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:14:08.722 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-22T23:14:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:14:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:14:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=178 2026-02-22T23:14:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 180 = 178 2026-02-22T23:14:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:14:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T23:14:09.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:09.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:09.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-22T23:14:09.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:14:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:14:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=180 2026-02-22T23:14:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 180 = 180 2026-02-22T23:14:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:09.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_resync_image 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+replaying 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:09.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:09.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+replaying '' '' 2026-02-22T23:14:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T23:14:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 2d0ac1d1-d0c0-4648-bbba-c710a4ad1078 2026-02-22T23:14:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802043,"replay_state":"idle"} 2026-02-22T23:14:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:04 2026-02-22T23:14:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:14:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:05' 2026-02-22T23:14:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:14:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:14:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:14:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802043,"replay_state":"idle"} 2026-02-22T23:14:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:14:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:14:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:10.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:10.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:10.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:04' 2026-02-22T23:14:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+replaying '2026-02-22 23:14:04' 2026-02-22T23:14:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:04' 2026-02-22T23:14:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK256305635mirror.0trial08219.2.3-689-gf90f4eaetrueOK56375642mirror.1trial08219.2.3-689-gf90f4eaefalseOK56445648mirror.2trial08219.2.3-689-gf90f4eaefalseOK56505654mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}56305635mirror.0trial0822026-02-22 23:13:48cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:14:10test_resync_image2d0ac1d1-d0c0-4648-bbba-c710a4ad1078up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802043,"replay_state":"idle"}56375642mirror.1trial0822026-02-22 23:14:04cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:14:05' 2026-02-22T23:14:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:04' 2026-02-22T23:14:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:14:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:14:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:14:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:14:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+stopped 2026-02-22T23:14:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:11.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+stopped '' '' 2026-02-22T23:14:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:11.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T23:14:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 2d0ac1d1-d0c0-4648-bbba-c710a4ad1078 2026-02-22T23:14:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:10 2026-02-22T23:14:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:11.705 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:14:11.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:11.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802043,"replay_state":"idle"} 2026-02-22T23:14:11.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:04 2026-02-22T23:14:11.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:14:11.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 178 .mirror.primary.2d0ac1d1-d0c0-4648-bbba-c710a4ad1078.50edc708-f777-474c-b31e-486efda94af2 (peer_uuids:[]) 2026-02-22T23:14:11.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 180 .mirror.primary.2d0ac1d1-d0c0-4648-bbba-c710a4ad1078.7e001de5-61bb-46b5-8ca2-ed198b15ab93 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:14:11.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:14:11.707 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:14:11.707 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:14:11.707 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:14:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:14:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:14:11.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:11.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:11.710 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:11.710 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:10' 2026-02-22T23:14:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+stopped '2026-02-22 23:14:10' 2026-02-22T23:14:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:10' 2026-02-22T23:14:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK244464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:14:10cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}2026-02-22 23:13:48test_resync_image2d0ac1d1-d0c0-4648-bbba-c710a4ad1078up+stoppedlocal image is primary44534458mirror.1trial0822026-02-22 23:14:10cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802043,"replay_state":"idle"}2026-02-22 23:14:04' 2026-02-22T23:14:11.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:10' 2026-02-22T23:14:11.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:14:11.746 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:14:11.746 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:14:11.746 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:14:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-22T23:14:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:14:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:14:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:14:11.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:14:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:14:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:14:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 9091 bytes/sec: 36 MiB/s 2026-02-22T23:14:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test_resync_image 2026-02-22T23:14:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_resync_image 2026-02-22T23:14:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:14:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ request_resync_image cluster1 mirror test_resync_image test_resync_image_id 2026-02-22T23:14:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id_var_name=test_resync_image_id 2026-02-22T23:14:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test_resync_image 2026-02-22T23:14:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:14:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_resync_image 2026-02-22T23:14:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test_resync_image 2026-02-22T23:14:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:14:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'test_resync_image_id='\''160ad8b0b112'\''' 2026-02-22T23:14:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test_resync_image_id=160ad8b0b112 2026-02-22T23:14:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'test -n "$test_resync_image_id"' 2026-02-22T23:14:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -n 160ad8b0b112 2026-02-22T23:14:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test_resync_image 2026-02-22T23:14:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Flagged image for resync from primary 2026-02-22T23:14:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-22T23:14:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T23:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T23:14:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 2d0ac1d1-d0c0-4648-bbba-c710a4ad1078 2026-02-22T23:14:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:10 2026-02-22T23:14:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:14:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:14' 2026-02-22T23:14:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:14:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:14:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:14:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T23:14:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:16.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:16.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T23:14:16.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:16.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 2d0ac1d1-d0c0-4648-bbba-c710a4ad1078 2026-02-22T23:14:16.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:16.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:15 2026-02-22T23:14:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:14:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:14' 2026-02-22T23:14:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-22T23:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:14:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:16.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:15' 2026-02-22T23:14:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+unknown '2026-02-22 23:14:15' 2026-02-22T23:14:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:15' 2026-02-22T23:14:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING1144464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:14:15cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}2026-02-22 23:14:14test_resync_image2d0ac1d1-d0c0-4648-bbba-c710a4ad1078up+unknownremote image is not primary44534458mirror.1trial0822026-02-22 23:14:15cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:14:14' 2026-02-22T23:14:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:15' 2026-02-22T23:14:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:14:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:14:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:14:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:14:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-22T23:14:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T23:14:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 2d0ac1d1-d0c0-4648-bbba-c710a4ad1078 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:14 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:15' 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:14:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-22T23:14:17.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:14:17.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:17.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:14:17.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:17.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:14:17.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:17.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:14' 2026-02-22T23:14:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+unknown '2026-02-22 23:14:14' 2026-02-22T23:14:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:14' 2026-02-22T23:14:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING1156305635mirror.0trial08219.2.3-689-gf90f4eaetrueOK56375642mirror.1trial08219.2.3-689-gf90f4eaefalseOK56445648mirror.2trial08219.2.3-689-gf90f4eaefalseOK56505654mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}56305635mirror.0trial0822026-02-22 23:14:14cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:14:15test_resync_image2d0ac1d1-d0c0-4648-bbba-c710a4ad1078up+unknownremote image is not primary56375642mirror.1trial0822026-02-22 23:14:14cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:14:15' 2026-02-22T23:14:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:14' 2026-02-22T23:14:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:14:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:14:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:14:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:14:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test_resync_image 2026-02-22T23:14:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:14:17.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_resync_image 2026-02-22T23:14:18.013 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:14:18.018 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test_resync_image 2026-02-22T23:14:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image started 2026-02-22T23:14:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:14:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:14:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:18.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-02-22T23:14:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:14:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:14:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:14:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:14:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:14:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:14:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:14:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:14:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:19.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:14:19.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:14:19.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:14:19.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:14:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:14:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:14:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-02-22T23:14:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:14:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:14:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:14:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:14:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:14:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:14:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:14:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:14:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:14:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:14:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:14:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:14:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:14:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:21.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:21.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:21.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:21.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:21.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:14:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:14:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:14:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:14:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:14:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:14:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:14:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:14:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+stopped 2026-02-22T23:14:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-02-22T23:14:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:22.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T23:14:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 2d0ac1d1-d0c0-4648-bbba-c710a4ad1078 2026-02-22T23:14:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:19 2026-02-22T23:14:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:14:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802058,"replay_state":"idle"} 2026-02-22T23:14:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:20 2026-02-22T23:14:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:14:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 185 .mirror.primary.2d0ac1d1-d0c0-4648-bbba-c710a4ad1078.32e01b18-7575-4dcd-9454-d49a0bdac912 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:14:22.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:14:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:14:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:14:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:14:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:14:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:14:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:19' 2026-02-22T23:14:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+stopped '2026-02-22 23:14:19' 2026-02-22T23:14:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:19' 2026-02-22T23:14:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:22.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK256305635mirror.0trial08219.2.3-689-gf90f4eaetrueOK56375642mirror.1trial08219.2.3-689-gf90f4eaefalseOK56445648mirror.2trial08219.2.3-689-gf90f4eaefalseOK56505654mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}56305635mirror.0trial0822026-02-22 23:14:14cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:14:20test_resync_image2d0ac1d1-d0c0-4648-bbba-c710a4ad1078up+stoppedlocal image is primary56375642mirror.1trial0822026-02-22 23:14:19cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802058,"replay_state":"idle"}2026-02-22 23:14:20' 2026-02-22T23:14:22.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:22.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:19' 2026-02-22T23:14:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:14:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:14:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+replaying 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:23.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+replaying '' '' 2026-02-22T23:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:23.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T23:14:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 2d0ac1d1-d0c0-4648-bbba-c710a4ad1078 2026-02-22T23:14:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802058,"replay_state":"idle"} 2026-02-22T23:14:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:20 2026-02-22T23:14:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:14:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:19' 2026-02-22T23:14:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:14:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:14:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:14:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802058,"replay_state":"idle"} 2026-02-22T23:14:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:14:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:14:23.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:23.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:23.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:23.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:23.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:20' 2026-02-22T23:14:23.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+replaying '2026-02-22 23:14:20' 2026-02-22T23:14:23.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:20' 2026-02-22T23:14:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:23.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK244464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:14:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}2026-02-22 23:14:14test_resync_image2d0ac1d1-d0c0-4648-bbba-c710a4ad1078up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802058,"replay_state":"idle"}44534458mirror.1trial0822026-02-22 23:14:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:14:19' 2026-02-22T23:14:23.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:23.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:20' 2026-02-22T23:14:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:14:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:14:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:14:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:14:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test_resync_image 2026-02-22T23:14:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:14:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_resync_image.export 2026-02-22T23:14:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_resync_image.export 2026-02-22T23:14:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:14:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.export 2026-02-22T23:14:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_resync_image.export 2026-02-22T23:14:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_resync_image.export 2026-02-22T23:14:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:14:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.export 2026-02-22T23:14:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.export 2026-02-22T23:14:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test_resync_image /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.export 2026-02-22T23:14:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:14:23.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test_resync_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.export 2026-02-22T23:14:23.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:14:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.export 2026-02-22T23:14:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.export 2026-02-22T23:14:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster1 mirror test_resync_image 2026-02-22T23:14:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster1 mirror test_resync_image 2026-02-22T23:14:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_resync_image 2026-02-22T23:14:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 rm mirror/test_resync_image 2026-02-22T23:14:24.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:14:24.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:24.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z '' ']' 2026-02-22T23:14:24.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: request image resync when remote is not primary and daemon is offline' 2026-02-22T23:14:24.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:14:24.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:14:24 TEST: request image resync when remote is not primary and daemon is offline 2026-02-22T23:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:14:24 TEST: request image resync when remote is not primary and daemon is offline 2026-02-22T23:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_resync_image=test_resync_image 2026-02-22T23:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-22T23:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test_resync_image 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:14:24.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:14:24.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:24.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/test_resync_image 2026-02-22T23:14:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:14:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:14:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-22T23:14:24.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:24.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:24.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:24.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:14:24.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_resync_image snapshot 2026-02-22T23:14:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:14:25.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test_resync_image 2026-02-22T23:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_resync_image': 2026-02-22T23:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 1733821dce18 2026-02-22T23:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.1733821dce18 2026-02-22T23:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:14:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:14:24 2026 2026-02-22T23:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:14:24 2026 2026-02-22T23:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:14:24 2026 2026-02-22T23:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:14:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-22T23:14:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:14:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:14:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:14:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:14:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:14:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:14:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 11111.2 bytes/sec: 43 MiB/s 2026-02-22T23:14:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_resync_image 2026-02-22T23:14:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-22T23:14:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:14:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:14:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-22T23:14:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:14:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:14:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:14:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:14:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:14:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:14:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:14:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:14:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:14:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:14:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:26.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:26.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:14:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:14:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:14:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:26.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:26.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:2 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:14:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:2 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:2 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:26.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T23:14:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:14:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:14:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T23:14:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:14:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:14:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:14:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:3 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:3 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:3 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:14:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:14:26.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T23:14:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:14:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:14:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:14:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:14:27.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:14:27.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:14:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:14:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:14:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:14:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:14:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:27.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:27.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:27.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:14:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:27.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:14:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:14:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:14:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:14:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:14:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:14:27.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_resync_image 2026-02-22T23:14:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_resync_image started 2026-02-22T23:14:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:27.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:27.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:27.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:14:27.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:14:27.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:27.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_resync_image started 2026-02-22T23:14:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:14:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:14:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:14:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:14:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:14:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:14:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:14:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:14:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:14:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:14:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:14:28.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:14:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:14:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:14:28.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:14:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:14:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:14:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:14:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:28.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:28.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:28.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:28.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:14:28.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:14:28.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:28.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:28.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:14:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:28.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:28.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:28.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:28.523 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:14:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:28.525 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:14:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:14:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:14:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:14:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:14:28.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:14:28.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:14:28.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:14:28.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_resync_image 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test_resync_image 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_resync_image 2026-02-22T23:14:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 184 2026-02-22T23:14:29.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_resync_image 2026-02-22T23:14:29.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:14:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_resync_image.status 2026-02-22T23:14:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_resync_image.status 2026-02-22T23:14:29.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:14:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.status 2026-02-22T23:14:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_resync_image.status 2026-02-22T23:14:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_resync_image.status 2026-02-22T23:14:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:14:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_resync_image /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.status 2026-02-22T23:14:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:29.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:29.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.status 2026-02-22T23:14:29.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_resync_image --format xml 2026-02-22T23:14:29.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:14:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:14:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=184 2026-02-22T23:14:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:14:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:14:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:14:29.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:29.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:29.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:29.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:29.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:29.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-22T23:14:29.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:14:29.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:14:29.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=183 2026-02-22T23:14:29.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 184 = 183 2026-02-22T23:14:29.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:14:29.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:14:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-22T23:14:29.710 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:14:29.943 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:14:29.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=183 2026-02-22T23:14:29.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 184 = 183 2026-02-22T23:14:29.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:14:29.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T23:14:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.status 2026-02-22T23:14:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-22T23:14:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:14:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:14:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=184 2026-02-22T23:14:30.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 184 = 184 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_resync_image 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+replaying 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:31.779 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+replaying '' '' 2026-02-22T23:14:31.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:31.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:31.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:31.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:31.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:31.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:31.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:31.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:31.780 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T23:14:31.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:31.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:31.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:31.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802065,"replay_state":"idle"} 2026-02-22T23:14:31.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:31.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:26 2026-02-22T23:14:31.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:31.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:14:31.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:31.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:31.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:30' 2026-02-22T23:14:31.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:14:31.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:14:31.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:14:31.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802065,"replay_state":"idle"} 2026-02-22T23:14:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:14:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:31.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:14:31.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:31.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:31.815 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:31.816 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:31.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:26' 2026-02-22T23:14:31.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+replaying '2026-02-22 23:14:26' 2026-02-22T23:14:31.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:31.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:31.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:31.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:31.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:26' 2026-02-22T23:14:31.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:31.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:31.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:31.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK256305635mirror.0trial08219.2.3-689-gf90f4eaetrueOK56375642mirror.1trial08219.2.3-689-gf90f4eaefalseOK56445648mirror.2trial08219.2.3-689-gf90f4eaefalseOK56505654mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}56305635mirror.0trial0822026-02-22 23:14:14cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:14:30test_resync_image0388b71f-1ffb-4330-9a06-64275165790dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802065,"replay_state":"idle"}56375642mirror.1trial0822026-02-22 23:14:26cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:14:30' 2026-02-22T23:14:31.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:31.851 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:31.852 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:26' 2026-02-22T23:14:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:14:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:14:31.855 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:14:31.856 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+stopped 2026-02-22T23:14:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+stopped '' '' 2026-02-22T23:14:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T23:14:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:30 2026-02-22T23:14:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:14:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802065,"replay_state":"idle"} 2026-02-22T23:14:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:26 2026-02-22T23:14:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:14:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 183 .mirror.primary.0388b71f-1ffb-4330-9a06-64275165790d.5e835329-061e-43a4-95a3-8ccbf38e2dfa (peer_uuids:[]) 2026-02-22T23:14:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 184 .mirror.primary.0388b71f-1ffb-4330-9a06-64275165790d.79d9ffbb-7d76-4638-9eac-6b959698cbe6 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:14:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:14:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:14:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:14:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:14:32.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:32.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:14:32.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:32.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:14:32.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:32.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:32.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:32.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:30' 2026-02-22T23:14:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+stopped '2026-02-22 23:14:30' 2026-02-22T23:14:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:30' 2026-02-22T23:14:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK244464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:14:30cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}2026-02-22 23:14:14test_resync_image0388b71f-1ffb-4330-9a06-64275165790dup+stoppedlocal image is primary44534458mirror.1trial0822026-02-22 23:14:30cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802065,"replay_state":"idle"}2026-02-22 23:14:26' 2026-02-22T23:14:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:30' 2026-02-22T23:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:14:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:14:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:14:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:14:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirrors cluster1 2026-02-22T23:14:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:14:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:14:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T23:14:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:14:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:14:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:14:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T23:14:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:0 2026-02-22T23:14:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:14:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:14:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:14:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:14:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:14:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:14:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:14:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:14:32.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=76955 2026-02-22T23:14:32.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 76955 ']' 2026-02-22T23:14:32.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 76955 2026-02-22T23:14:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=76955 '$2 == pid {print; exit 1}' 2026-02-22T23:14:32.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 76955 0.2 0.0 670916 47388 ? Rsl 23:13 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=76955 '$2 == pid {print; exit 1}' 2026-02-22T23:14:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:14:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=76955 '$2 == pid {print; exit 1}' 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:14:33.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:33.995 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T23:14:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:14:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:14:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:14:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:14:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:0 2026-02-22T23:14:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:14:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:14:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:14:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:14:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:14:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:14:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T23:14:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:14:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:14:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:14:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:1 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:14:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:14:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:14:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=76998 2026-02-22T23:14:34.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 76998 ']' 2026-02-22T23:14:34.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 76998 2026-02-22T23:14:34.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:34.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:34.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:34.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=76998 '$2 == pid {print; exit 1}' 2026-02-22T23:14:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 76998 0.4 0.0 736480 52416 ? Ssl 23:13 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=76998 '$2 == pid {print; exit 1}' 2026-02-22T23:14:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:14:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=76998 '$2 == pid {print; exit 1}' 2026-02-22T23:14:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:14:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:14:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:14:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:14:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:14:35.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:35.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:14:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:1 2026-02-22T23:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:14:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:14:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:14:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T23:14:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:14:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:14:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:2 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:14:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:14:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:14:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:14:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:14:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:14:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:14:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:14:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=77060 2026-02-22T23:14:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 77060 ']' 2026-02-22T23:14:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 77060 2026-02-22T23:14:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:35.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:35.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=77060 '$2 == pid {print; exit 1}' 2026-02-22T23:14:35.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 77060 0.1 0.0 670916 45988 ? Ssl 23:13 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=77060 '$2 == pid {print; exit 1}' 2026-02-22T23:14:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:14:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=77060 '$2 == pid {print; exit 1}' 2026-02-22T23:14:36.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:14:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:14:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:14:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:14:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:14:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:14:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:14:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:14:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:14:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:14:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T23:14:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:14:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:14:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:14:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:14:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:14:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:14:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:14:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:14:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:14:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:2 2026-02-22T23:14:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:14:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:14:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:14:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:14:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:14:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:14:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:14:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:14:36.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:14:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T23:14:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:14:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:14:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:14:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T23:14:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:3 2026-02-22T23:14:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:14:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:14:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:14:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:14:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:14:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:14:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:14:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:14:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:14:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:14:36.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:14:36.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:14:36.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=77120 2026-02-22T23:14:36.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 77120 ']' 2026-02-22T23:14:36.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 77120 2026-02-22T23:14:36.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:36.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:36.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=77120 '$2 == pid {print; exit 1}' 2026-02-22T23:14:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 77120 0.1 0.0 670916 44580 ? Ssl 23:13 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=77120 '$2 == pid {print; exit 1}' 2026-02-22T23:14:37.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:14:37.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:37.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=77120 '$2 == pid {print; exit 1}' 2026-02-22T23:14:37.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:14:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:14:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:14:37.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:14:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T23:14:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:14:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:14:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:14:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:37.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:14:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:14:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:14:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:14:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:14:37.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:14:37.282 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:3 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:14:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:14:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:14:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-22T23:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:14:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:14:37.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:14:37.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:14:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test_resync_image 2026-02-22T23:14:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_resync_image 2026-02-22T23:14:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:14:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ request_resync_image cluster1 mirror test_resync_image test_resync_image_id 2026-02-22T23:14:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id_var_name=test_resync_image_id 2026-02-22T23:14:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test_resync_image 2026-02-22T23:14:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:14:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_resync_image 2026-02-22T23:14:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test_resync_image 2026-02-22T23:14:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:14:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'test_resync_image_id='\''160a9a5073c3'\''' 2026-02-22T23:14:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test_resync_image_id=160a9a5073c3 2026-02-22T23:14:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'test -n "$test_resync_image_id"' 2026-02-22T23:14:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -n 160a9a5073c3 2026-02-22T23:14:39.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test_resync_image 2026-02-22T23:14:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Flagged image for resync from primary 2026-02-22T23:14:39.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirrors cluster1 2026-02-22T23:14:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:14:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:0 2026-02-22T23:14:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:14:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:14:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:14:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T23:14:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:14:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:14:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:14:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T23:14:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:14:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T23:14:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:1 2026-02-22T23:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:14:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:2 2026-02-22T23:14:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:14:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:14:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:14:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T23:14:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T23:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T23:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T23:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:39.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:3 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T23:14:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:14:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T23:14:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:14:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:39.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:39.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-22T23:14:39.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:39.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:39.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:39.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:39.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T23:14:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T23:14:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:35 2026-02-22T23:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:14:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:14:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:14:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:14:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T23:14:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T23:14:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:40 2026-02-22T23:14:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:14:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:14:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:14:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:14:42.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:14:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:14:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-22T23:14:42.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:14:42.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:42.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:14:42.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:42.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:14:42.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:42.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:42.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:42.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:40' 2026-02-22T23:14:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+unknown '2026-02-22 23:14:40' 2026-02-22T23:14:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:40' 2026-02-22T23:14:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING244464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:14:40cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112down+unknownstatus not foundtest_resync_image0388b71f-1ffb-4330-9a06-64275165790dup+unknownremote image is not primary44534458mirror.1trial0822026-02-22 23:14:40cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112down+unknownstatus not found' 2026-02-22T23:14:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:42.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:40' 2026-02-22T23:14:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:14:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:14:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:14:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:14:42.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:42.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:42.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-22T23:14:42.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T23:14:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T23:14:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771802069,"remote_snapshot_timestamp":1771802078,"replay_state":"idle"} 2026-02-22T23:14:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:43 2026-02-22T23:14:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:14:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:40' 2026-02-22T23:14:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:14:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:14:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:14:45.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T23:14:45.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:45.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:45.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:45.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T23:14:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image demoted 2026-02-22T23:14:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:14:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:45 2026-02-22T23:14:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:14:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:40' 2026-02-22T23:14:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:14:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:14:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-22T23:14:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:14:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:14:49.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-22T23:14:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-22T23:14:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T23:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:14:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:49 2026-02-22T23:14:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:14:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:45' 2026-02-22T23:14:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:14:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:14:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-22T23:14:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:14:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:14:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:14:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:14:49.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:49.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:49.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:49' 2026-02-22T23:14:49.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+unknown '2026-02-22 23:14:49' 2026-02-22T23:14:49.572 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:14:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:49' 2026-02-22T23:14:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:49.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING1157625767mirror.0trial08219.2.3-689-gf90f4eaetrueOK57695774mirror.1trial08219.2.3-689-gf90f4eaefalseOK57765783mirror.2trial08219.2.3-689-gf90f4eaefalseOK57855788mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}57695774mirror.1trial0822026-02-22 23:14:43cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:14:45test_resync_image0388b71f-1ffb-4330-9a06-64275165790dup+unknownremote image is not primary57625767mirror.0trial0822026-02-22 23:14:49cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:14:45' 2026-02-22T23:14:49.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:49.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:49.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:49' 2026-02-22T23:14:49.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:49.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:14:49.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:14:49.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:14:49.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:14:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test_resync_image 2026-02-22T23:14:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:14:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_resync_image 2026-02-22T23:14:50.028 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:14:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test_resync_image 2026-02-22T23:14:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image started 2026-02-22T23:14:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:14:50.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:14:50.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:50.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-02-22T23:14:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:14:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:14:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:14:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:14:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:14:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:14:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:14:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:14:51.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:14:51.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:14:51.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:51.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:14:51.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:51.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:51.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:51.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:51.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:14:51.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:51.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:51.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:14:51.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-22T23:14:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:14:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:14:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-22T23:14:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:14:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:14:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:14:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:14:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:14:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:14:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:14:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-22T23:14:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:14:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:14:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:14:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:14:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:14:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:14:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+stopped 2026-02-22T23:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-02-22T23:14:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:52.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T23:14:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:14:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:14:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:14:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:49 2026-02-22T23:14:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:14:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802090,"replay_state":"idle"} 2026-02-22T23:14:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:50 2026-02-22T23:14:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:14:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 189 .mirror.primary.0388b71f-1ffb-4330-9a06-64275165790d.817e1660-d86c-4b28-b055-ffc5f88d7afa (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:14:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:14:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:14:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:14:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-02-22T23:14:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-22T23:14:54.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:54.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:54 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802090,"replay_state":"idle"} 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:50 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:14:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 189 .mirror.primary.0388b71f-1ffb-4330-9a06-64275165790d.817e1660-d86c-4b28-b055-ffc5f88d7afa (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:14:54.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:14:54.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:14:54.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:14:54.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:14:54.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:54.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:14:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:14:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:14:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:54' 2026-02-22T23:14:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+stopped '2026-02-22 23:14:54' 2026-02-22T23:14:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:54.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:14:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:54' 2026-02-22T23:14:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK257625767mirror.0trial08219.2.3-689-gf90f4eaetrueOK57695774mirror.1trial08219.2.3-689-gf90f4eaefalseOK57765783mirror.2trial08219.2.3-689-gf90f4eaefalseOK57855788mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}57695774mirror.1trial0822026-02-22 23:14:43cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:14:50test_resync_image0388b71f-1ffb-4330-9a06-64275165790dup+stoppedlocal image is primary57625767mirror.0trial0822026-02-22 23:14:54cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802090,"replay_state":"idle"}2026-02-22 23:14:50' 2026-02-22T23:14:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:54' 2026-02-22T23:14:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:14:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:14:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:14:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+replaying 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:14:54.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+replaying '' '' 2026-02-22T23:14:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:14:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:14:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:55.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-22T23:14:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_resync_image: 2026-02-22T23:14:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 0388b71f-1ffb-4330-9a06-64275165790d 2026-02-22T23:14:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:14:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802090,"replay_state":"idle"} 2026-02-22T23:14:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:14:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:50 2026-02-22T23:14:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:14:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:14:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:14:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:14:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:14:54' 2026-02-22T23:14:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:14:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:14:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:14:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802090,"replay_state":"idle"} 2026-02-22T23:14:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:14:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:14:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:14:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:14:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:14:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:14:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:14:55.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:14:55.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:50' 2026-02-22T23:14:55.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+replaying '2026-02-22 23:14:50' 2026-02-22T23:14:55.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:14:55.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:55.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:55.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:14:55.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:14:50' 2026-02-22T23:14:55.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:14:55.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:14:55.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:14:55.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK244464451mirror.0trial08219.2.3-689-gf90f4eaetrueOK44534458mirror.1trial08219.2.3-689-gf90f4eaefalseOK44604463mirror.2trial08219.2.3-689-gf90f4eaefalseOK44704478mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestd73b653c-f1e8-487c-91fe-8129b425b941up+stoppedlocal image is primary44464451mirror.0trial0822026-02-22 23:14:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771801870,"remote_snapshot_timestamp":1771801870,"replay_state":"idle"}2026-02-22 23:14:43test_resync_image0388b71f-1ffb-4330-9a06-64275165790dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802090,"replay_state":"idle"}44534458mirror.1trial0822026-02-22 23:14:50cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:14:54' 2026-02-22T23:14:55.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:14:55.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-22T23:14:55.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:14:50' 2026-02-22T23:14:55.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-22T23:14:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:14:55.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:14:55.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:14:55.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:14:55.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:55.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:55.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test_resync_image 2026-02-22T23:14:55.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:14:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_resync_image.export 2026-02-22T23:14:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_resync_image.export 2026-02-22T23:14:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:14:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.export 2026-02-22T23:14:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_resync_image.export 2026-02-22T23:14:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_resync_image.export 2026-02-22T23:14:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:14:55.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.export 2026-02-22T23:14:55.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.export 2026-02-22T23:14:55.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test_resync_image /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.export 2026-02-22T23:14:55.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:14:55.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test_resync_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.export 2026-02-22T23:14:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:14:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.export 2026-02-22T23:14:55.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_resync_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_resync_image.export 2026-02-22T23:14:55.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster1 mirror test_resync_image 2026-02-22T23:14:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster1 mirror test_resync_image 2026-02-22T23:14:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:14:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_resync_image 2026-02-22T23:14:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_resync_image 2026-02-22T23:14:55.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 rm mirror/test_resync_image 2026-02-22T23:14:56.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:14:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:14:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: disable mirror while daemon is stopped' 2026-02-22T23:14:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:14:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:14:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:14:56 TEST: disable mirror while daemon is stopped 2026-02-22T23:14:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:14:56 TEST: disable mirror while daemon is stopped 2026-02-22T23:14:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirrors cluster1 2026-02-22T23:14:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:14:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:14:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:0 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:14:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:14:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:14:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:14:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:14:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:14:56.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:14:56.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:14:56.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:14:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=79240 2026-02-22T23:14:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 79240 ']' 2026-02-22T23:14:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 79240 2026-02-22T23:14:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79240 '$2 == pid {print; exit 1}' 2026-02-22T23:14:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 79240 0.5 0.0 736480 47656 ? Ssl 23:14 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:57.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:57.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79240 '$2 == pid {print; exit 1}' 2026-02-22T23:14:57.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:14:57.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:57.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79240 '$2 == pid {print; exit 1}' 2026-02-22T23:14:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:14:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:14:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:14:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:14:57.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:14:57.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:57.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T23:14:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:14:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:14:57.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:57.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:14:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:0 2026-02-22T23:14:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:14:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:14:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:14:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:57.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:14:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:14:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:14:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:14:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:14:57.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:14:57.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:14:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T23:14:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:14:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:14:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:14:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:1 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:14:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:14:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:14:57.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:14:57.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:14:57.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=79279 2026-02-22T23:14:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 79279 ']' 2026-02-22T23:14:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 79279 2026-02-22T23:14:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79279 '$2 == pid {print; exit 1}' 2026-02-22T23:14:57.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 79279 0.4 0.0 736488 46044 ? Ssl 23:14 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:57.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:57.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79279 '$2 == pid {print; exit 1}' 2026-02-22T23:14:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:14:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79279 '$2 == pid {print; exit 1}' 2026-02-22T23:14:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:14:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:14:58.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:58.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T23:14:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:14:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:58.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:14:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:14:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:1 2026-02-22T23:14:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:14:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:14:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:14:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:14:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:14:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:14:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:14:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:14:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:14:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:14:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:2 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:58.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:58.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:14:58.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:14:58.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:14:58.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:14:58.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:14:58.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:14:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:14:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:14:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=79344 2026-02-22T23:14:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 79344 ']' 2026-02-22T23:14:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 79344 2026-02-22T23:14:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:58.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79344 '$2 == pid {print; exit 1}' 2026-02-22T23:14:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 79344 0.3 0.0 670912 44304 ? Ssl 23:14 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:14:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:59.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79344 '$2 == pid {print; exit 1}' 2026-02-22T23:14:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:14:59.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:59.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79344 '$2 == pid {print; exit 1}' 2026-02-22T23:14:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:14:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:14:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:14:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:14:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:14:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:14:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T23:14:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:14:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:14:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:14:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:14:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:14:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:14:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:2 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:59.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:14:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:14:59.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:14:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:14:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:14:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:14:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T23:14:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:14:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:14:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:14:59.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:14:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T23:14:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:3 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:14:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:14:59.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:14:59.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:14:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:14:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:14:59.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=79422 2026-02-22T23:14:59.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 79422 ']' 2026-02-22T23:14:59.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 79422 2026-02-22T23:14:59.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:59.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:14:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:14:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79422 '$2 == pid {print; exit 1}' 2026-02-22T23:14:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 79422 0.3 0.0 670920 43164 ? Ssl 23:14 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:14:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:14:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:00.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:00.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79422 '$2 == pid {print; exit 1}' 2026-02-22T23:15:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:15:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=79422 '$2 == pid {print; exit 1}' 2026-02-22T23:15:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:15:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:15:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:15:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:15:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:15:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:15:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T23:15:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:15:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:15:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:15:00.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:00.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:3 2026-02-22T23:15:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:15:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:15:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:15:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:15:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:15:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:15:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:15:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirrors cluster2 2026-02-22T23:15:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:15:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:15:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster2:0 '' 2026-02-22T23:15:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:0 2026-02-22T23:15:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:15:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster2:0 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2:0 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster2 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster2 2026-02-22T23:15:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:15:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:15:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T23:15:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T23:15:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T23:15:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=53887 2026-02-22T23:15:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 53887 ']' 2026-02-22T23:15:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 53887 2026-02-22T23:15:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=53887 '$2 == pid {print; exit 1}' 2026-02-22T23:15:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 53887 0.2 0.0 744684 63260 ? Ssl 22:59 0:02 rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=53887 '$2 == pid {print; exit 1}' 2026-02-22T23:15:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:15:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=53887 '$2 == pid {print; exit 1}' 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:0 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:01.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:15:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:15:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:15:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:01.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:15:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:15:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:15:01.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:15:01.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:0 2026-02-22T23:15:01.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:15:01.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:01.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:15:01.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:15:01.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:01.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:01.482 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:01.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:01.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:15:01.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:15:01.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:15:01.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:01.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:01.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:15:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:15:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:15:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster2:0 2026-02-22T23:15:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:15:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:15:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:15:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:15:01.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:15:01.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:15:01.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:01.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:01.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T23:15:01.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T23:15:01.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T23:15:01.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster2:1 '' 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:1 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster2:1 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2:1 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:15:01.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:15:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:15:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:15:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:15:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster2 2026-02-22T23:15:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster2 2026-02-22T23:15:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:15:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:15:01.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T23:15:01.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T23:15:01.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T23:15:01.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=53931 2026-02-22T23:15:01.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 53931 ']' 2026-02-22T23:15:01.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 53931 2026-02-22T23:15:01.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:01.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:01.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:01.521 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=53931 '$2 == pid {print; exit 1}' 2026-02-22T23:15:01.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 53931 0.1 0.0 736488 61904 ? Ssl 22:59 0:01 rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:01.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:01.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:02.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:02.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=53931 '$2 == pid {print; exit 1}' 2026-02-22T23:15:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:15:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=53931 '$2 == pid {print; exit 1}' 2026-02-22T23:15:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-02-22T23:15:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:1 2026-02-22T23:15:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:15:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:15:02.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:15:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:15:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:15:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:15:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:15:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:15:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:15:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:15:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:15:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:15:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:1 2026-02-22T23:15:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:15:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:15:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:15:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:15:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:15:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:15:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:15:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:15:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:15:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:15:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:15:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:15:02.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster2:1 2026-02-22T23:15:02.583 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:15:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:15:02.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T23:15:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T23:15:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T23:15:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster2:2 '' 2026-02-22T23:15:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:2 2026-02-22T23:15:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:15:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:15:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster2:2 2026-02-22T23:15:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2:2 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:2 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster2 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster2 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:15:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:15:02.598 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T23:15:02.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T23:15:02.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T23:15:02.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=53991 2026-02-22T23:15:02.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 53991 ']' 2026-02-22T23:15:02.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 53991 2026-02-22T23:15:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=53991 '$2 == pid {print; exit 1}' 2026-02-22T23:15:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 53991 0.0 0.0 670920 57140 ? Ssl 22:59 0:00 rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:02.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=53991 '$2 == pid {print; exit 1}' 2026-02-22T23:15:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:15:03.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=53991 '$2 == pid {print; exit 1}' 2026-02-22T23:15:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-02-22T23:15:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:2 2026-02-22T23:15:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T23:15:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:15:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:15:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:15:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:15:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:15:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:15:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:15:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T23:15:03.657 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:15:03.657 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:15:03.660 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T23:15:03.660 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:2 2026-02-22T23:15:03.660 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:15:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T23:15:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:15:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:15:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:15:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:15:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:15:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:15:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:15:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T23:15:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:15:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:15:03.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster2:2 2026-02-22T23:15:03.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:2 2026-02-22T23:15:03.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:03.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:15:03.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:15:03.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:03.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:03.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:03.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:15:03.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:15:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:15:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:15:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:15:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:15:03.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T23:15:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T23:15:03.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T23:15:03.689 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:03.689 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster2:3 '' 2026-02-22T23:15:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:3 2026-02-22T23:15:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:15:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:15:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster2:3 2026-02-22T23:15:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2:3 2026-02-22T23:15:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:15:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:3 2026-02-22T23:15:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:15:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:15:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T23:15:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster2 2026-02-22T23:15:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster2 2026-02-22T23:15:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:15:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:15:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T23:15:03.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T23:15:03.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T23:15:03.699 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=54060 2026-02-22T23:15:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 54060 ']' 2026-02-22T23:15:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 54060 2026-02-22T23:15:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=54060 '$2 == pid {print; exit 1}' 2026-02-22T23:15:03.710 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 54060 0.0 0.0 670920 57144 ? Ssl 22:59 0:00 rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=54060 '$2 == pid {print; exit 1}' 2026-02-22T23:15:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:15:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=54060 '$2 == pid {print; exit 1}' 2026-02-22T23:15:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-02-22T23:15:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:3 2026-02-22T23:15:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T23:15:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:15:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:15:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:15:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:15:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:15:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T23:15:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:15:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:15:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T23:15:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:3 2026-02-22T23:15:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:15:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T23:15:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:15:04.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:15:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:15:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:15:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:15:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T23:15:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:15:04.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:15:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster2:3 2026-02-22T23:15:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:3 2026-02-22T23:15:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:15:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:15:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:15:04.763 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:15:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:15:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:15:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:15:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T23:15:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T23:15:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T23:15:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:15:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ disable_mirror cluster2 mirror test 2026-02-22T23:15:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test 2026-02-22T23:15:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring disabled 2026-02-22T23:15:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z '' ']' 2026-02-22T23:15:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test present 2026-02-22T23:15:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:15:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:05.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=10e6d7f7515 2026-02-22T23:15:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 10e6d7f7515 2026-02-22T23:15:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z '' -o '' = 10e6d7f7515 2026-02-22T23:15:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:15:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = present 2026-02-22T23:15:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirrors cluster1 2026-02-22T23:15:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:15:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:0 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T23:15:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:1 2026-02-22T23:15:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:15:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:15:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T23:15:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T23:15:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T23:15:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T23:15:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:15:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:2 2026-02-22T23:15:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:15:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:3 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:05.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id=10e6d7f7515 2026-02-22T23:15:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:15:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted 10e6d7f7515 2026-02-22T23:15:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=10e6d7f7515 2026-02-22T23:15:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:05.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=10e6d7f7515 2026-02-22T23:15:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 10e6d7f7515 2026-02-22T23:15:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 10e6d7f7515 -o 10e6d7f7515 = 10e6d7f7515 2026-02-22T23:15:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:15:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = present 2026-02-22T23:15:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:06.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted 10e6d7f7515 2026-02-22T23:15:06.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=10e6d7f7515 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=10e6d7f7515 2026-02-22T23:15:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 10e6d7f7515 2026-02-22T23:15:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 10e6d7f7515 -o 10e6d7f7515 = 10e6d7f7515 2026-02-22T23:15:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:15:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = present 2026-02-22T23:15:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted 10e6d7f7515 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=10e6d7f7515 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=10e6d7f7515 2026-02-22T23:15:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 10e6d7f7515 2026-02-22T23:15:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 10e6d7f7515 -o 10e6d7f7515 = 10e6d7f7515 2026-02-22T23:15:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:15:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = present 2026-02-22T23:15:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:15:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted 10e6d7f7515 2026-02-22T23:15:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=10e6d7f7515 2026-02-22T23:15:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:12.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:15:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:12.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:15:12.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:12.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:15:12.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test 2026-02-22T23:15:12.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:12.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:15:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-22T23:15:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:15:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-22T23:15:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test': 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 114b5d8fbe2c 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.114b5d8fbe2c 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 22:59:09 2026 2026-02-22T23:15:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:00:15 2026 2026-02-22T23:15:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:00:24 2026 2026-02-22T23:15:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:15:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:15:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 60914a81-d952-4381-b228-36f34f786ddc 2026-02-22T23:15:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:15:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T23:15:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=present 2026-02-22T23:15:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:13.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id= 2026-02-22T23:15:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:15:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T23:15:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:13.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:15:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = deleted 2026-02-22T23:15:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:13.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:14.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T23:15:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:15:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:14.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:14.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:14.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:14.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:15:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = deleted 2026-02-22T23:15:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:15:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T23:15:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:15:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=16cfd3f41690 2026-02-22T23:15:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 16cfd3f41690 2026-02-22T23:15:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z '' -o '' = 16cfd3f41690 2026-02-22T23:15:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = present 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:15:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:17.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:15:17.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:17.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:17.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:17.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:15:17.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:15:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:15:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:15:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:15:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:15:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:15:17.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:15:17.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:17.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:15:17.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:17.895 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:15:17.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:17.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:15:17.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:15:17.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:17.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:17.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:17.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:15:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:15:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:15:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:15:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:17.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:17.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:17.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:17.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:15:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:15:17.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:15:17.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:15:17.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:15:17.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:15:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:15:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:15:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: non-default namespace image mirroring' 2026-02-22T23:15:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:15:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:15:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:15:17 TEST: non-default namespace image mirroring 2026-02-22T23:15:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:15:17 TEST: non-default namespace image mirroring 2026-02-22T23:15:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog ' - replay' 2026-02-22T23:15:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:15:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:15:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:15:17 - replay 2026-02-22T23:15:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:15:17 - replay 2026-02-22T23:15:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test snapshot 2026-02-22T23:15:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror/ns1 test 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:17.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:15:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:15:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/ns1/test 2026-02-22T23:15:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-22T23:15:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:15:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror/ns1 test snapshot 2026-02-22T23:15:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:15:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns1/test snapshot 2026-02-22T23:15:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:15:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/ns1/test 2026-02-22T23:15:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test': 2026-02-22T23:15:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:15:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:15:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:15:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 178be4102d70 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.178be4102d70 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:15:17 2026 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:15:17 2026 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:15:17 2026 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: c0756102-2795-4722-b8fe-d25d584ebca8 2026-02-22T23:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:15:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns2 test snapshot 2026-02-22T23:15:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror/ns2 test 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:15:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:15:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/ns2/test 2026-02-22T23:15:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror/ns2 2026-02-22T23:15:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:15:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror/ns2 test snapshot 2026-02-22T23:15:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:15:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns2/test snapshot 2026-02-22T23:15:19.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:15:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/ns2/test 2026-02-22T23:15:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test': 2026-02-22T23:15:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:15:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:15:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:15:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 17948aa4e9dc 2026-02-22T23:15:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.17948aa4e9dc 2026-02-22T23:15:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:15:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:15:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:15:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:15:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:15:18 2026 2026-02-22T23:15:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:15:18 2026 2026-02-22T23:15:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:15:18 2026 2026-02-22T23:15:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:15:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:15:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: b5846924-16c1-4a3c-ac6a-f126d2c6a39a 2026-02-22T23:15:19.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:15:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test 2026-02-22T23:15:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test started 2026-02-22T23:15:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:15:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror/ns1 test started 2026-02-22T23:15:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test 2026-02-22T23:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:15:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:15:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:15:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:15:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-02-22T23:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:15:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:15:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-02-22T23:15:20.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:15:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:15:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:15:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:15:20.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:15:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:15:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:15:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror/ns2 test 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror/ns2 test started 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror/ns2 test started 2026-02-22T23:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns2/test 2026-02-22T23:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:15:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:15:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:15:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:15:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns2/test 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:21.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:21.156 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:15:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:15:21.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:21.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:21.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:21.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns2/test 2026-02-22T23:15:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:15:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:15:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:15:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:15:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:15:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:15:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:15:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror/ns1 test 100 2026-02-22T23:15:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:15:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:15:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:15:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:15:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:15:21.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:15:21.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 11111.2 bytes/sec: 43 MiB/s 2026-02-22T23:15:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror/ns2 test 100 2026-02-22T23:15:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:15:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:15:21.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:21.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:15:21.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/ns2/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:15:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:15:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:15:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 5555.61 bytes/sec: 22 MiB/s 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 test 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror/ns1 test 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns1/test 2026-02-22T23:15:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 190 2026-02-22T23:15:21.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns1 test 2026-02-22T23:15:21.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:15:21.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:21.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:21.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror/ns1-test.status 2026-02-22T23:15:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror/ns1-test.status 2026-02-22T23:15:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:21.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test.status 2026-02-22T23:15:21.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror/ns1-test.status 2026-02-22T23:15:21.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror/ns1-test.status 2026-02-22T23:15:21.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:21.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test.status 2026-02-22T23:15:21.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 test /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test.status 2026-02-22T23:15:21.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:21.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:21.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:21.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test.status 2026-02-22T23:15:21.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/test --format xml 2026-02-22T23:15:21.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:15:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:15:21.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=190 2026-02-22T23:15:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:15:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:15:21.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:15:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test.status 2026-02-22T23:15:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test.status 2026-02-22T23:15:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-02-22T23:15:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:15:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:15:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=188 2026-02-22T23:15:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 190 = 188 2026-02-22T23:15:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:15:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:15:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test.status 2026-02-22T23:15:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test.status 2026-02-22T23:15:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:15:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-02-22T23:15:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:15:22.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=190 2026-02-22T23:15:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 190 = 190 2026-02-22T23:15:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns2 test 2026-02-22T23:15:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:15:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:15:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:15:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror/ns2 test 2026-02-22T23:15:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns2/test 2026-02-22T23:15:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 191 2026-02-22T23:15:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns2 test 2026-02-22T23:15:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:15:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror/ns2-test.status 2026-02-22T23:15:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror/ns2-test.status 2026-02-22T23:15:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror_ns2-test.status 2026-02-22T23:15:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror/ns2-test.status 2026-02-22T23:15:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror/ns2-test.status 2026-02-22T23:15:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:22.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns2-test.status 2026-02-22T23:15:22.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns2 test /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns2-test.status 2026-02-22T23:15:22.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:22.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:22.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:22.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror_ns2-test.status 2026-02-22T23:15:22.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns2/test --format xml 2026-02-22T23:15:22.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:15:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:15:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=191 2026-02-22T23:15:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:15:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:15:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:15:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns2 test /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns2-test.status 2026-02-22T23:15:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns2-test.status 2026-02-22T23:15:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns2/test --format xml 2026-02-22T23:15:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:15:22.674 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:15:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=189 2026-02-22T23:15:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 191 = 189 2026-02-22T23:15:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:15:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:15:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns2 test /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns2-test.status 2026-02-22T23:15:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns2-test.status 2026-02-22T23:15:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns2/test --format xml 2026-02-22T23:15:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:15:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:15:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=191 2026-02-22T23:15:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 191 = 191 2026-02-22T23:15:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test 2026-02-22T23:15:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:15:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test up+replaying 2026-02-22T23:15:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:15:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:15:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test up+replaying '' '' 2026-02-22T23:15:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:15:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:15:24.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:15:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:15:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: c0756102-2795-4722-b8fe-d25d584ebca8 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802118,"replay_state":"idle"} 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:15:20 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:15:24.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:15:24.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:15:24.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:15:24.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:15:24.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802118,"replay_state":"idle"} 2026-02-22T23:15:24.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:15:24.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:15:24.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:15:24.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:15:24.147 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:15:24.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:15:24.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:15:24.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:15:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:20' 2026-02-22T23:15:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test up+replaying '2026-02-22 23:15:20' 2026-02-22T23:15:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:15:20' 2026-02-22T23:15:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:15:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:15:24.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-02-22T23:15:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING158345839mirror.0trial08219.2.3-689-gf90f4eaetrueOK58415845mirror.1trial08219.2.3-689-gf90f4eaefalseOK58475851mirror.2trial08219.2.3-689-gf90f4eaefalseOK58535865mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestc0756102-2795-4722-b8fe-d25d584ebca8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802118,"replay_state":"idle"}58345839mirror.0trial0822026-02-22 23:15:20cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:15:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:15:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:15:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:20' 2026-02-22T23:15:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:15:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:15:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:15:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:15:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:15:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns2 test 2026-02-22T23:15:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:15:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns2 test up+replaying 2026-02-22T23:15:24.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:24.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:24.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:24.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:24.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:15:24.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:15:24.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:24.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:25.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror/ns2 test up+replaying '' '' 2026-02-22T23:15:25.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:15:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:15:25.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:15:25.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:15:25.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns2/test 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: b5846924-16c1-4a3c-ac6a-f126d2c6a39a 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802119,"replay_state":"idle"} 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:15:21 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:15:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:15:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:15:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:15:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:15:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802119,"replay_state":"idle"} 2026-02-22T23:15:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:15:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:15:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:15:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:15:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:15:25.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:15:25.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:15:25.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:15:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:21' 2026-02-22T23:15:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns2 test up+replaying '2026-02-22 23:15:21' 2026-02-22T23:15:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:15:21' 2026-02-22T23:15:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:15:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:15:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns2 --verbose --format xml 2026-02-22T23:15:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING158345839mirror.0trial08219.2.3-689-gf90f4eaetrueOK58415845mirror.1trial08219.2.3-689-gf90f4eaefalseOK58475851mirror.2trial08219.2.3-689-gf90f4eaefalseOK58535865mirror.3trial08219.2.3-689-gf90f4eaefalseOKtestb5846924-16c1-4a3c-ac6a-f126d2c6a39aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802119,"replay_state":"idle"}58345839mirror.0trial0822026-02-22 23:15:21cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:15:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:15:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:15:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:21' 2026-02-22T23:15:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:15:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:15:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:15:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:15:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:15:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror/ns1 test 2026-02-22T23:15:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:15:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror/ns1-test.export 2026-02-22T23:15:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror/ns1-test.export 2026-02-22T23:15:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test.export 2026-02-22T23:15:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror/ns1-test.export 2026-02-22T23:15:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror/ns1-test.export 2026-02-22T23:15:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:25.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test.export 2026-02-22T23:15:25.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test.export 2026-02-22T23:15:25.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/ns1/test /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test.export 2026-02-22T23:15:25.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:15:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/ns1/test /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test.export 2026-02-22T23:15:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:15:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test.export 2026-02-22T23:15:25.715 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test.export 2026-02-22T23:15:25.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:25.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror/ns2 test 2026-02-22T23:15:25.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:25.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:25.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:15:25.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror/ns2-test.export 2026-02-22T23:15:25.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror/ns2-test.export 2026-02-22T23:15:25.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:25.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror_ns2-test.export 2026-02-22T23:15:25.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror/ns2-test.export 2026-02-22T23:15:25.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror/ns2-test.export 2026-02-22T23:15:25.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:25.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns2-test.export 2026-02-22T23:15:25.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns2-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns2-test.export 2026-02-22T23:15:25.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/ns2/test /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns2-test.export 2026-02-22T23:15:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:15:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/ns2/test /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns2-test.export 2026-02-22T23:15:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:15:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns2-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns2-test.export 2026-02-22T23:15:26.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns2-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns2-test.export 2026-02-22T23:15:26.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog ' - disable mirroring / delete image' 2026-02-22T23:15:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:15:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:15:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:15:26 - disable mirroring / delete image 2026-02-22T23:15:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:15:26 - disable mirroring / delete image 2026-02-22T23:15:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror/ns1 test 2026-02-22T23:15:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:26.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror/ns1 test 2026-02-22T23:15:26.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:26.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:26.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:26.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test 2026-02-22T23:15:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test 2026-02-22T23:15:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:15:27.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:27.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ disable_mirror cluster2 mirror/ns2 test 2026-02-22T23:15:27.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:27.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:27.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:27.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/ns2/test 2026-02-22T23:15:28.017 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring disabled 2026-02-22T23:15:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror/ns1 test deleted 2026-02-22T23:15:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:15:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-02-22T23:15:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror/ns1 2026-02-22T23:15:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-02-22T23:15:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:28.048 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id= 2026-02-22T23:15:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:15:28.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror/ns1 test deleted '' 2026-02-22T23:15:28.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:28.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:28.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:28.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:28.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:28.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:28.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:28.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-02-22T23:15:28.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror/ns1 2026-02-22T23:15:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-02-22T23:15:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:15:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:15:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror/ns2 test deleted 2026-02-22T23:15:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:15:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror/ns2 test 2026-02-22T23:15:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror/ns2 2026-02-22T23:15:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/ns2/test 2026-02-22T23:15:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:28.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:28.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id= 2026-02-22T23:15:28.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:28.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:15:28.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror/ns2 test deleted '' 2026-02-22T23:15:28.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:28.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror/ns2 test 2026-02-22T23:15:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror/ns2 2026-02-22T23:15:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/ns2/test 2026-02-22T23:15:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:15:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:15:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror/ns2 test 2026-02-22T23:15:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror/ns2 test 2026-02-22T23:15:28.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:28.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns2 2026-02-22T23:15:28.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:28.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns2/test 2026-02-22T23:15:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/ns2/test 2026-02-22T23:15:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:15:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog ' - data pool' 2026-02-22T23:15:28.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:15:28.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:15:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:15:28 - data pool 2026-02-22T23:15:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:15:28 - data pool 2026-02-22T23:15:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ dp_image=test_data_pool 2026-02-22T23:15:28.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test_data_pool snapshot 128 --data-pool mirror_parent 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror/ns1 test_data_pool 128 --data-pool mirror_parent 2026-02-22T23:15:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:15:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 128 ']' 2026-02-22T23:15:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=128 2026-02-22T23:15:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock --data-pool mirror_parent mirror/ns1/test_data_pool 2026-02-22T23:15:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-22T23:15:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:15:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror/ns1 test_data_pool snapshot 2026-02-22T23:15:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:15:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns1/test_data_pool snapshot 2026-02-22T23:15:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:15:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/ns1/test_data_pool 2026-02-22T23:15:29.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_data_pool': 2026-02-22T23:15:29.051 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:15:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:15:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:15:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 17c438b34d5e 2026-02-22T23:15:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: data_pool: mirror_parent 2026-02-22T23:15:29.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.3.17c438b34d5e 2026-02-22T23:15:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:15:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, data-pool 2026-02-22T23:15:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:15:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:15:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:15:28 2026 2026-02-22T23:15:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:15:28 2026 2026-02-22T23:15:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:15:28 2026 2026-02-22T23:15:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:15:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:15:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 376d66be-5a84-4f42-83e2-a3118ba4820f 2026-02-22T23:15:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:15:29.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_data_pool cluster2 mirror/ns1 test_data_pool 2026-02-22T23:15:29.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:15:29.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror/ns1 2026-02-22T23:15:29.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_data_pool 2026-02-22T23:15:29.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 info mirror/ns1/test_data_pool 2026-02-22T23:15:29.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T23:15:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ data_pool=mirror_parent 2026-02-22T23:15:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test mirror_parent = mirror_parent 2026-02-22T23:15:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test_data_pool 2026-02-22T23:15:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test_data_pool started 2026-02-22T23:15:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:15:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:30.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror/ns1 test_data_pool started 2026-02-22T23:15:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:15:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:15:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:15:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:15:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:15:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:15:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:15:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:15:30.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:15:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:30.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:15:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:30.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T23:15:30.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:15:30.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:15:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:15:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:15:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:15:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:15:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:15:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:15:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:15:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:15:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:15:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:30.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T23:15:30.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:15:30.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:15:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:15:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:15:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:15:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:15:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:15:30.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:15:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:15:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:15:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T23:15:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:15:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:15:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:30.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:15:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:15:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:15:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T23:15:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:15:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:15:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:15:31.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:31.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:15:31.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:31.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:31.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:15:31.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:31.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:31.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:31.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:15:31.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:15:31.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T23:15:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:15:31.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:15:31.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:15:31.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:15:31.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:15:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:15:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:15:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_data_pool cluster1 mirror/ns1 test_data_pool 2026-02-22T23:15:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror/ns1 2026-02-22T23:15:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_data_pool 2026-02-22T23:15:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 info mirror/ns1/test_data_pool 2026-02-22T23:15:31.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T23:15:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ data_pool=mirror_parent 2026-02-22T23:15:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test mirror_parent = mirror_parent 2026-02-22T23:15:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror/ns1 test_data_pool 100 2026-02-22T23:15:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:15:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:15:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:15:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:15:31.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:15:31.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:15:31.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:15:31.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 test_data_pool 2026-02-22T23:15:31.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:15:31.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:31.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:15:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:15:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror/ns1 test_data_pool 2026-02-22T23:15:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns1/test_data_pool 2026-02-22T23:15:32.027 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 17 2026-02-22T23:15:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns1 test_data_pool 2026-02-22T23:15:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:15:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror/ns1-test_data_pool.status 2026-02-22T23:15:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror/ns1-test_data_pool.status 2026-02-22T23:15:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T23:15:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror/ns1-test_data_pool.status 2026-02-22T23:15:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror/ns1-test_data_pool.status 2026-02-22T23:15:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T23:15:32.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 test_data_pool /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T23:15:32.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:32.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:32.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:32.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T23:15:32.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-22T23:15:32.037 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:15:32.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:15:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=17 2026-02-22T23:15:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:15:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:15:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:15:32.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T23:15:32.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:32.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:32.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:32.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T23:15:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-22T23:15:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:15:32.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:15:32.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=16 2026-02-22T23:15:32.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 17 = 16 2026-02-22T23:15:32.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:15:32.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:15:32.700 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T23:15:32.700 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T23:15:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-22T23:15:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:15:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:15:32.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=16 2026-02-22T23:15:32.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 17 = 16 2026-02-22T23:15:32.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:15:32.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T23:15:33.534 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T23:15:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.status 2026-02-22T23:15:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-22T23:15:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:15:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:15:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=17 2026-02-22T23:15:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 17 = 17 2026-02-22T23:15:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test_data_pool 2026-02-22T23:15:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:15:33.566 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test_data_pool up+replaying 2026-02-22T23:15:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:15:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:15:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test_data_pool up+replaying '' '' 2026-02-22T23:15:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:15:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:15:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:15:34.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:15:34.568 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test_data_pool 2026-02-22T23:15:34.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_data_pool: 2026-02-22T23:15:34.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 376d66be-5a84-4f42-83e2-a3118ba4820f 2026-02-22T23:15:34.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:15:34.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802129,"replay_state":"idle"} 2026-02-22T23:15:34.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:15:34.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:15:31 2026-02-22T23:15:34.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:15:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:15:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:15:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:15:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:15:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:15:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:15:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:15:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802129,"replay_state":"idle"} 2026-02-22T23:15:34.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:15:34.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:15:34.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:15:34.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:15:34.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:15:34.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:15:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:15:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:15:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:31' 2026-02-22T23:15:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test_data_pool up+replaying '2026-02-22 23:15:31' 2026-02-22T23:15:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:15:31' 2026-02-22T23:15:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:15:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:15:34.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-02-22T23:15:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING158345839mirror.0trial08219.2.3-689-gf90f4eaetrueOK58415845mirror.1trial08219.2.3-689-gf90f4eaefalseOK58475851mirror.2trial08219.2.3-689-gf90f4eaefalseOK58535865mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_data_pool376d66be-5a84-4f42-83e2-a3118ba4820fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802129,"replay_state":"idle"}58345839mirror.0trial0822026-02-22 23:15:31cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:15:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:15:34.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/last_update' 2026-02-22T23:15:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:31' 2026-02-22T23:15:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/state' 2026-02-22T23:15:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:15:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:15:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:15:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:15:34.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:34.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:34.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror/ns1 test_data_pool 2026-02-22T23:15:34.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:15:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror/ns1-test_data_pool.export 2026-02-22T23:15:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror/ns1-test_data_pool.export 2026-02-22T23:15:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test_data_pool.export 2026-02-22T23:15:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror/ns1-test_data_pool.export 2026-02-22T23:15:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror/ns1-test_data_pool.export 2026-02-22T23:15:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T23:15:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T23:15:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/ns1/test_data_pool /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test_data_pool.export 2026-02-22T23:15:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:15:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/ns1/test_data_pool /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T23:15:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:15:34.823 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T23:15:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.VQMrvUqClS/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T23:15:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror/ns1 test_data_pool 2026-02-22T23:15:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:34.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:34.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:34.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror/ns1 test_data_pool 2026-02-22T23:15:34.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:15:34.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror/ns1 2026-02-22T23:15:34.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_data_pool 2026-02-22T23:15:34.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test_data_pool 2026-02-22T23:15:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test_data_pool 2026-02-22T23:15:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:15:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: simple image resync' 2026-02-22T23:15:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:15:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:15:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:15:35 TEST: simple image resync 2026-02-22T23:15:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:15:35 TEST: simple image resync 2026-02-22T23:15:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-22T23:15:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id_var_name=image_id 2026-02-22T23:15:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'image_id='\''16cfd3f41690'\''' 2026-02-22T23:15:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ image_id=16cfd3f41690 2026-02-22T23:15:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'test -n "$image_id"' 2026-02-22T23:15:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -n 16cfd3f41690 2026-02-22T23:15:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-22T23:15:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Flagged image for resync from primary 2026-02-22T23:15:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test deleted 16cfd3f41690 2026-02-22T23:15:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:15:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=16cfd3f41690 2026-02-22T23:15:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 16cfd3f41690 2026-02-22T23:15:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:15:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted 16cfd3f41690 2026-02-22T23:15:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=16cfd3f41690 2026-02-22T23:15:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:35.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:35.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:35.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:35.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:35.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:35.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:35.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:15:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:15:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T23:15:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=present 2026-02-22T23:15:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:35.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:35.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:35.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id= 2026-02-22T23:15:35.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:35.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:15:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T23:15:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:15:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = deleted 2026-02-22T23:15:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T23:15:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:15:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:36.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:36.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:36.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:36.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:36.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:36.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:15:36.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:36.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = deleted 2026-02-22T23:15:36.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:36.433 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:15:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T23:15:38.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:38.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:38.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:38.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:15:38.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:38.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:15:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = deleted 2026-02-22T23:15:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:38.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T23:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:42.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=16cf6c57286f 2026-02-22T23:15:42.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 16cf6c57286f 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z '' -o '' = 16cf6c57286f 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = present 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:42.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:42.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:42.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:15:42.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:42.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:42.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:15:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:15:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:15:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:15:43.497 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:15:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:15:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:15:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:15:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:15:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:15:43.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:15:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:15:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:15:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:43.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:15:43.504 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:15:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:15:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:15:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:15:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:15:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:15:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:15:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:15:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:15:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:15:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:15:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:15:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:15:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:15:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:15:44.593 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 60914a81-d952-4381-b228-36f34f786ddc 2026-02-22T23:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802113,"replay_state":"idle"} 2026-02-22T23:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:15:40 2026-02-22T23:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:15:44.594 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:15:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:15:44.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:15:44.596 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802113,"replay_state":"idle"} 2026-02-22T23:15:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:15:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:15:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:15:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:15:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:15:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:40' 2026-02-22T23:15:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:15:40' 2026-02-22T23:15:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:15:40' 2026-02-22T23:15:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:15:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:15:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:15:44.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING158345839mirror.0trial08219.2.3-689-gf90f4eaetrueOK58415845mirror.1trial08219.2.3-689-gf90f4eaefalseOK58475851mirror.2trial08219.2.3-689-gf90f4eaefalseOK58535865mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest60914a81-d952-4381-b228-36f34f786ddcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802113,"replay_state":"idle"}58345839mirror.0trial0822026-02-22 23:15:40cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:15:44.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:15:44.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:15:44.633 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:40' 2026-02-22T23:15:44.634 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:15:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:15:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:15:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:15:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:15:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test 2026-02-22T23:15:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:15:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T23:15:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.export 2026-02-22T23:15:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T23:15:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T23:15:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.export 2026-02-22T23:15:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:44.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:15:44.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:15:44.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T23:15:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:15:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:15:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:15:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:15:44.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:15:44.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:44.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z '' ']' 2026-02-22T23:15:44.908 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: image resync while replayer is stopped' 2026-02-22T23:15:44.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:15:44.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:15:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:15:44 TEST: image resync while replayer is stopped 2026-02-22T23:15:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:15:44 TEST: image resync while replayer is stopped 2026-02-22T23:15:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemons cluster1 rbd mirror stop mirror/test 2026-02-22T23:15:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_instance=cluster1 2026-02-22T23:15:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance=cluster1 2026-02-22T23:15:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loop_instance 2026-02-22T23:15:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:15:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror/test 2026-02-22T23:15:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:15:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T23:15:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:44.925 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:44.925 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:44.925 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:44.925 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror/test 2026-02-22T23:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T23:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T23:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:15:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T23:15:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:15:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:15:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:15:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:15:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:15:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:15:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:15:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:15:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:15:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:15:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:15:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:15:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:45.980 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:15:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:15:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:45.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:45.991 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:45.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:45.992 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:15:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:15:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:15:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:15:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:15:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:15:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-22T23:15:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id_var_name=image_id 2026-02-22T23:15:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:46.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'image_id='\''16cf6c57286f'\''' 2026-02-22T23:15:46.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ image_id=16cf6c57286f 2026-02-22T23:15:46.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'test -n "$image_id"' 2026-02-22T23:15:46.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -n 16cf6c57286f 2026-02-22T23:15:46.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-22T23:15:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Flagged image for resync from primary 2026-02-22T23:15:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-22T23:15:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_instance=cluster1 2026-02-22T23:15:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance=cluster1 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loop_instance 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:15:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:46.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:46.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-22T23:15:46.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test deleted 16cf6c57286f 2026-02-22T23:15:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:15:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=16cf6c57286f 2026-02-22T23:15:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 16cf6c57286f 2026-02-22T23:15:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:15:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted 16cf6c57286f 2026-02-22T23:15:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:46.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=16cf6c57286f 2026-02-22T23:15:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:46.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T23:15:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id= 2026-02-22T23:15:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:15:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-22T23:15:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_instance=cluster1 2026-02-22T23:15:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:46.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:46.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance=cluster1 2026-02-22T23:15:46.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loop_instance 2026-02-22T23:15:46.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:46.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:46.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:46.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:15:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-22T23:15:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:15:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:15:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:46.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-22T23:15:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=present 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:46.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:46.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id=16cfb04747c 2026-02-22T23:15:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:15:46.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test present 16cfb04747c 2026-02-22T23:15:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:15:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=16cfb04747c 2026-02-22T23:15:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:46.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:46.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:46.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:46.480 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:46.505 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=16cfb04747c 2026-02-22T23:15:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 16cfb04747c 2026-02-22T23:15:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 16cfb04747c -o 16cfb04747c = 16cfb04747c 2026-02-22T23:15:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:15:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = present 2026-02-22T23:15:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:15:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:46.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:15:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:15:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:46.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:47.505 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:15:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:15:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:15:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:15:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:15:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:15:47.506 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:15:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:15:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:15:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:15:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:15:47.508 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:15:47.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:15:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:15:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:15:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:15:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:15:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:47.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:47.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:47.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:47.522 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:15:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:15:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:15:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:15:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:15:47.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:15:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:15:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:15:48.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:15:48.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:15:48.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:15:48.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:15:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:15:48.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 60914a81-d952-4381-b228-36f34f786ddc 2026-02-22T23:15:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:15:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802113,"replay_state":"idle"} 2026-02-22T23:15:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:15:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:15:46 2026-02-22T23:15:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:15:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:15:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:15:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:15:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:15:48.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:15:48.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:15:48.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:15:48.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802113,"replay_state":"idle"} 2026-02-22T23:15:48.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:15:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:15:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:15:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:15:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:15:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:15:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:15:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:15:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:46' 2026-02-22T23:15:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:15:46' 2026-02-22T23:15:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:15:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:15:46' 2026-02-22T23:15:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:15:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:15:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:15:48.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING158345839mirror.0trial08219.2.3-689-gf90f4eaetrueOK58415845mirror.1trial08219.2.3-689-gf90f4eaefalseOK58475851mirror.2trial08219.2.3-689-gf90f4eaefalseOK58535865mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest60914a81-d952-4381-b228-36f34f786ddcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802113,"replay_state":"idle"}58345839mirror.0trial0822026-02-22 23:15:46cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:15:48.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:15:48.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:15:48.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:46' 2026-02-22T23:15:48.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:15:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:15:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:15:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:15:48.646 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:15:48.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:48.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:48.647 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test 2026-02-22T23:15:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:15:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T23:15:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.export 2026-02-22T23:15:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T23:15:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T23:15:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.export 2026-02-22T23:15:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:15:48.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:15:48.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:15:48.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T23:15:48.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:15:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:15:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:15:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:15:48.882 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:15:48.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:15:48.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: request image resync while daemon is offline' 2026-02-22T23:15:48.914 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:15:48.914 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:15:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:15:48 TEST: request image resync while daemon is offline 2026-02-22T23:15:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:15:48 TEST: request image resync while daemon is offline 2026-02-22T23:15:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirrors cluster1 2026-02-22T23:15:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:15:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:15:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T23:15:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:15:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:15:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:0 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:15:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:15:48.918 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:15:48.930 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:15:48.930 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:15:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=80312 2026-02-22T23:15:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 80312 ']' 2026-02-22T23:15:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 80312 2026-02-22T23:15:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:48.932 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:48.932 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80312 '$2 == pid {print; exit 1}' 2026-02-22T23:15:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 80312 0.8 0.0 744680 58508 ? Ssl 23:15 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:49.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:49.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80312 '$2 == pid {print; exit 1}' 2026-02-22T23:15:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:15:49.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:49.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80312 '$2 == pid {print; exit 1}' 2026-02-22T23:15:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:15:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:15:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:49.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:49.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:49.971 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:49.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:49.981 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T23:15:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:15:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:49.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:49.985 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:15:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:15:49.996 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:0 2026-02-22T23:15:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:15:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:15:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:15:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:15:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:15:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:15:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:15:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T23:15:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:15:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:15:50.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:15:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T23:15:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:1 2026-02-22T23:15:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:15:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:15:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:15:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:50.010 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:15:50.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:15:50.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:15:50.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:15:50.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:15:50.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:15:50.011 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:15:50.020 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:15:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:15:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=80347 2026-02-22T23:15:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 80347 ']' 2026-02-22T23:15:50.022 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 80347 2026-02-22T23:15:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80347 '$2 == pid {print; exit 1}' 2026-02-22T23:15:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 80347 0.2 0.0 670916 42904 ? Rsl 23:15 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80347 '$2 == pid {print; exit 1}' 2026-02-22T23:15:51.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:15:51.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:51.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80347 '$2 == pid {print; exit 1}' 2026-02-22T23:15:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:15:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:15:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:15:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:15:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:15:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:15:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:15:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:15:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:15:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:15:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:15:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:15:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:15:51.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:15:51.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:15:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:1 2026-02-22T23:15:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:15:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:15:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:15:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:15:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:15:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:15:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:15:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:15:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:15:51.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:15:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T23:15:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:15:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:15:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:51.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:15:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T23:15:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:2 2026-02-22T23:15:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:15:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:15:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:15:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:15:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:15:51.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:15:51.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:15:51.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:15:51.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:15:51.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:15:51.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:15:51.110 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:15:51.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=80416 2026-02-22T23:15:51.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 80416 ']' 2026-02-22T23:15:51.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 80416 2026-02-22T23:15:51.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:51.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:51.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80416 '$2 == pid {print; exit 1}' 2026-02-22T23:15:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 80416 0.2 0.0 466020 44640 ? Ssl 23:15 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80416 '$2 == pid {print; exit 1}' 2026-02-22T23:15:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:15:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:52.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80416 '$2 == pid {print; exit 1}' 2026-02-22T23:15:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:15:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:15:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:15:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:15:52.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:15:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:15:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:15:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:15:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:15:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:15:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:15:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T23:15:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:15:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:15:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:15:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:15:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:15:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:15:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:15:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:15:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:15:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:15:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:15:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:2 2026-02-22T23:15:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:15:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:15:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:15:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:15:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:15:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:15:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:15:52.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:15:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:15:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:15:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T23:15:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:15:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:15:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:15:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T23:15:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:3 2026-02-22T23:15:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:15:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:15:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:15:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:15:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:15:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:15:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:15:52.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:15:52.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:15:52.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:15:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:15:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:15:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=80478 2026-02-22T23:15:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 80478 ']' 2026-02-22T23:15:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 80478 2026-02-22T23:15:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:15:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80478 '$2 == pid {print; exit 1}' 2026-02-22T23:15:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:ubuntu 80478 0.2 0.0 670916 43560 ? Ssl 23:15 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:15:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80478 '$2 == pid {print; exit 1}' 2026-02-22T23:15:53.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:15:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:15:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=80478 '$2 == pid {print; exit 1}' 2026-02-22T23:15:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:15:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:15:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:15:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:15:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:15:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:15:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T23:15:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:15:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:15:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:15:53.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:53.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:15:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:3 2026-02-22T23:15:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:15:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:15:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:15:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:15:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:15:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:15:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:53.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:53.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:15:53.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:15:53.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:15:53.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:15:53.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:15:53.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:15:53.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:15:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-22T23:15:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id_var_name=image_id 2026-02-22T23:15:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'image_id='\''16cfb04747c'\''' 2026-02-22T23:15:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ image_id=16cfb04747c 2026-02-22T23:15:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'test -n "$image_id"' 2026-02-22T23:15:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -n 16cfb04747c 2026-02-22T23:15:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-22T23:15:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Flagged image for resync from primary 2026-02-22T23:15:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirrors cluster1 2026-02-22T23:15:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:15:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:0 2026-02-22T23:15:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:15:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:15:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T23:15:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T23:15:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T23:15:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T23:15:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:15:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:53.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:53.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:1 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T23:15:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:15:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:2 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T23:15:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:15:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:3 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:15:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:15:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test deleted 16cfb04747c 2026-02-22T23:15:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=deleted 2026-02-22T23:15:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=16cfb04747c 2026-02-22T23:15:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:15:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 16cfb04747c 2026-02-22T23:15:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:15:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted 16cfb04747c 2026-02-22T23:15:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=16cfb04747c 2026-02-22T23:15:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=16cfb04747c 2026-02-22T23:15:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 16cfb04747c 2026-02-22T23:15:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 16cfb04747c -o 16cfb04747c = 16cfb04747c 2026-02-22T23:15:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:15:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = present 2026-02-22T23:15:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:15:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted 16cfb04747c 2026-02-22T23:15:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=16cfb04747c 2026-02-22T23:15:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:54.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=16cfb04747c 2026-02-22T23:15:54.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 16cfb04747c 2026-02-22T23:15:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 16cfb04747c -o 16cfb04747c = 16cfb04747c 2026-02-22T23:15:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:15:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = present 2026-02-22T23:15:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:15:56.588 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted 16cfb04747c 2026-02-22T23:15:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:15:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:15:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:15:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:15:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=16cfb04747c 2026-02-22T23:15:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:15:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:15:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:15:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:15:56.589 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:15:56.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:15:56.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:15:56.590 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:15:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=16cfb04747c 2026-02-22T23:15:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 16cfb04747c 2026-02-22T23:15:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 16cfb04747c -o 16cfb04747c = 16cfb04747c 2026-02-22T23:15:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:15:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = present 2026-02-22T23:15:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:15:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:16:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test deleted 16cfb04747c 2026-02-22T23:16:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=deleted 2026-02-22T23:16:00.619 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=16cfb04747c 2026-02-22T23:16:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:16:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:16:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:16:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:16:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:16:00.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:16:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=1779272f8b2f 2026-02-22T23:16:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 1779272f8b2f 2026-02-22T23:16:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 16cfb04747c -o 16cfb04747c = 1779272f8b2f 2026-02-22T23:16:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test deleted = deleted 2026-02-22T23:16:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T23:16:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=present 2026-02-22T23:16:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id= 2026-02-22T23:16:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:16:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:16:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:16:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:16:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:16:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image_id=1779272f8b2f 2026-02-22T23:16:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:16:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:16:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_present cluster1 mirror test present 1779272f8b2f 2026-02-22T23:16:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=present 2026-02-22T23:16:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id=1779272f8b2f 2026-02-22T23:16:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=deleted 2026-02-22T23:16:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_image_id 2026-02-22T23:16:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror test 2026-02-22T23:16:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test 2026-02-22T23:16:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T23:16:00.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:16:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_image_id=1779272f8b2f 2026-02-22T23:16:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 1779272f8b2f 2026-02-22T23:16:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -z 1779272f8b2f -o 1779272f8b2f = 1779272f8b2f 2026-02-22T23:16:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=present 2026-02-22T23:16:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test present = present 2026-02-22T23:16:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T23:16:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T23:16:00.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:16:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:16:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T23:16:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:16:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:16:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:16:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T23:16:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:16:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:16:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:16:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:16:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:16:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:16:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:16:01.817 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:01.818 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:16:01.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:16:01.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:16:01.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:16:01.819 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:16:01.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:01.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:01.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:01.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T23:16:01.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:16:01.874 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:16:01.875 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:16:01.875 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:16:01.876 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:16:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:16:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:16:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T23:16:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:01.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T23:16:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T23:16:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test: 2026-02-22T23:16:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 60914a81-d952-4381-b228-36f34f786ddc 2026-02-22T23:16:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:16:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802113,"replay_state":"idle"} 2026-02-22T23:16:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:16:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:15:58 2026-02-22T23:16:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:16:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:16:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:16:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:16:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:16:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:16:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:16:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:16:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802113,"replay_state":"idle"} 2026-02-22T23:16:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:16:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:16:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:16:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:16:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:16:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:16:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:16:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:16:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:58' 2026-02-22T23:16:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 23:15:58' 2026-02-22T23:16:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:15:58' 2026-02-22T23:16:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:02.917 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:16:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING160046009mirror.0trial08219.2.3-689-gf90f4eaetrueOK60116016mirror.1trial08219.2.3-689-gf90f4eaefalseOK60186022mirror.2trial08219.2.3-689-gf90f4eaefalseOK60266032mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest60914a81-d952-4381-b228-36f34f786ddcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802113,"replay_state":"idle"}60046009mirror.0trial0822026-02-22 23:15:58cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:16:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:16:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T23:16:02.948 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:15:58' 2026-02-22T23:16:02.949 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T23:16:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:16:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:16:02.950 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:16:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:16:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test 2026-02-22T23:16:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:16:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T23:16:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test.export 2026-02-22T23:16:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:16:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T23:16:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T23:16:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test.export 2026-02-22T23:16:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:16:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:16:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:16:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export 2026-02-22T23:16:03.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:16:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:16:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T23:16:03.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:16:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test.export 2026-02-22T23:16:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test 2026-02-22T23:16:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:16:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:03.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test 2026-02-22T23:16:03.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:03.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:03.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test 2026-02-22T23:16:03.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-22T23:16:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-22T23:16:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:16:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:16:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: split-brain' 2026-02-22T23:16:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:16:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:16:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:16:04 TEST: split-brain 2026-02-22T23:16:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:16:04 TEST: split-brain 2026-02-22T23:16:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ image=split-brain 2026-02-22T23:16:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror split-brain snapshot 2026-02-22T23:16:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror split-brain 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:04.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:16:04.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:16:04.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock mirror/split-brain 2026-02-22T23:16:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:16:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:16:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror split-brain snapshot 2026-02-22T23:16:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:16:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/split-brain snapshot 2026-02-22T23:16:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:16:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/split-brain 2026-02-22T23:16:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'split-brain': 2026-02-22T23:16:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 128 MiB in 32 objects 2026-02-22T23:16:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:16:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:16:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 18016b927c80 2026-02-22T23:16:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.18016b927c80 2026-02-22T23:16:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:16:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:16:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:16:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:16:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:16:04 2026 2026-02-22T23:16:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:16:04 2026 2026-02-22T23:16:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:16:04 2026 2026-02-22T23:16:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:16:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:16:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: d16dd329-25e2-4da5-8acb-642656d49280 2026-02-22T23:16:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:16:05.083 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror split-brain 2026-02-22T23:16:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:16:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+replaying 2026-02-22T23:16:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:06.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-22T23:16:06.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:06.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T23:16:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image split-brain: (2) No such file or directory 2026-02-22T23:16:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status= 2026-02-22T23:16:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:16:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:16:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:16:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-22T23:16:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T23:16:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='split-brain: 2026-02-22T23:16:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d16dd329-25e2-4da5-8acb-642656d49280 2026-02-22T23:16:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:16:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802165,"replay_state":"idle"} 2026-02-22T23:16:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:16:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:06 2026-02-22T23:16:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:16:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:16:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:16:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:16:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:16:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:16:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:16:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:16:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802165,"replay_state":"idle"} 2026-02-22T23:16:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:16:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:16:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:16:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:16:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:16:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:16:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:16:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:16:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:06' 2026-02-22T23:16:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+replaying '2026-02-22 23:16:06' 2026-02-22T23:16:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:16:06' 2026-02-22T23:16:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:08.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:16:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING160046009mirror.0trial08219.2.3-689-gf90f4eaetrueOK60116016mirror.1trial08219.2.3-689-gf90f4eaefalseOK60186022mirror.2trial08219.2.3-689-gf90f4eaefalseOK60266032mirror.3trial08219.2.3-689-gf90f4eaefalseOKsplit-braind16dd329-25e2-4da5-8acb-642656d49280up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802165,"replay_state":"idle"}60046009mirror.0trial0822026-02-22 23:16:06cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:16:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:16:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T23:16:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:06' 2026-02-22T23:16:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T23:16:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:16:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:16:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:16:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:16:08.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror split-brain --force 2026-02-22T23:16:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force=--force 2026-02-22T23:16:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/split-brain --force 2026-02-22T23:16:17.265 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:16:17.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror split-brain 2026-02-22T23:16:17.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:17.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:17.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:17.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror split-brain stopped 2026-02-22T23:16:17.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:17.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:17.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:17.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:16:17.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:16:17.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:17.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:18.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror split-brain stopped 2026-02-22T23:16:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:16:18.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:16:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:16:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/split-brain 2026-02-22T23:16:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:16:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:16:18.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:16:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:16:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:16:18.283 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:16:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:16:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/split-brain 2026-02-22T23:16:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:16:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:16:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:16:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:16:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:16:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:16:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:16:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:16:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:16:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:16:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:16:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:16:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:16:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:16:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:16:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:16:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:16:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/split-brain 2026-02-22T23:16:18.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:16:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:16:18.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:16:18.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:16:18.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:16:18.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:18.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+stopped 2026-02-22T23:16:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:16:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:19.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+stopped '' '' 2026-02-22T23:16:19.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:19.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:19.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:19.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:16:19.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:19.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:19.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:19.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:19.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T23:16:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='split-brain: 2026-02-22T23:16:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d16dd329-25e2-4da5-8acb-642656d49280 2026-02-22T23:16:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:16:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:16:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:16:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:18 2026-02-22T23:16:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:16:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:16:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:16:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:16:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22T23:16:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:16:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 200 .mirror.primary.d16dd329-25e2-4da5-8acb-642656d49280.8d2546d3-0abb-4644-939f-d6a43c8ccb5b (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:16:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:16:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:16:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:16:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:16:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:16:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:16:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:16:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:16:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:16:19.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:16:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:16:19.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:16:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:18' 2026-02-22T23:16:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+stopped '2026-02-22 23:16:18' 2026-02-22T23:16:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:16:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:16:18' 2026-02-22T23:16:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:16:19.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING160046009mirror.0trial08219.2.3-689-gf90f4eaetrueOK60116016mirror.1trial08219.2.3-689-gf90f4eaefalseOK60186022mirror.2trial08219.2.3-689-gf90f4eaefalseOK60266032mirror.3trial08219.2.3-689-gf90f4eaefalseOKsplit-braind16dd329-25e2-4da5-8acb-642656d49280up+stoppedlocal image is primary60046009mirror.0trial0822026-02-22 23:16:18cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:16:19.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:16:19.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T23:16:19.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:18' 2026-02-22T23:16:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T23:16:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:16:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:16:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:16:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:16:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster1 mirror split-brain 10 2026-02-22T23:16:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=10 2026-02-22T23:16:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:16:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:16:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 bench mirror/split-brain --io-type write --io-size 4096 --io-threads 1 --io-total 40960 --io-pattern rand 2026-02-22T23:16:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 40960 pattern random 2026-02-22T23:16:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:16:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 10 ops/sec: 5000.05 bytes/sec: 20 MiB/s 2026-02-22T23:16:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster1 mirror split-brain 2026-02-22T23:16:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/split-brain 2026-02-22T23:16:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:16:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+error split-brain 2026-02-22T23:16:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+error 2026-02-22T23:16:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern=split-brain 2026-02-22T23:16:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-22T23:16:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+error 2026-02-22T23:16:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern=split-brain 2026-02-22T23:16:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T23:16:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='split-brain: 2026-02-22T23:16:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d16dd329-25e2-4da5-8acb-642656d49280 2026-02-22T23:16:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:16:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:16:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:16:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:18 2026-02-22T23:16:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:16:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:16:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:16:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:16:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:16:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+error' 2026-02-22T23:16:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:16:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:16:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-22T23:16:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+error 2026-02-22T23:16:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern=split-brain 2026-02-22T23:16:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='split-brain: 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d16dd329-25e2-4da5-8acb-642656d49280 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:18 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:16:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+error' 2026-02-22T23:16:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:16:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:16:27.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-22T23:16:27.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:27.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:27.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:27.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+error 2026-02-22T23:16:27.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern=split-brain 2026-02-22T23:16:27.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:27.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:27.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:27.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T23:16:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='split-brain: 2026-02-22T23:16:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d16dd329-25e2-4da5-8acb-642656d49280 2026-02-22T23:16:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+error 2026-02-22T23:16:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: split-brain 2026-02-22T23:16:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:16:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:23 2026-02-22T23:16:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:16:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:16:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:16:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:16:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:16:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+error' 2026-02-22T23:16:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+error 2026-02-22T23:16:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*split-brain' 2026-02-22T23:16:27.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: split-brain 2026-02-22T23:16:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:16:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+error 2026-02-22T23:16:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:16:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+error 2026-02-22T23:16:27.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:16:27.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:23' 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+error '2026-02-22 23:16:23' 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+error 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:16:23' 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:16:27.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING160046009mirror.0trial08219.2.3-689-gf90f4eaetrueOK60116016mirror.1trial08219.2.3-689-gf90f4eaefalseOK60186022mirror.2trial08219.2.3-689-gf90f4eaefalseOK60266032mirror.3trial08219.2.3-689-gf90f4eaefalseOKsplit-braind16dd329-25e2-4da5-8acb-642656d49280up+errorsplit-brain60046009mirror.0trial0822026-02-22 23:16:23cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:16:27.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:16:27.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T23:16:27.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:23' 2026-02-22T23:16:27.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T23:16:27.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+error 2026-02-22T23:16:27.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+error 2026-02-22T23:16:27.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+error 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+error 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ request_resync_image cluster1 mirror split-brain image_id 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image_id_var_name=image_id 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_image_id cluster1 mirror split-brain 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:27.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=split-brain 2026-02-22T23:16:27.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster=cluster1 info mirror/split-brain 2026-02-22T23:16:27.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T23:16:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'image_id='\''17792dc168fd'\''' 2026-02-22T23:16:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ image_id=17792dc168fd 2026-02-22T23:16:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval 'test -n "$image_id"' 2026-02-22T23:16:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -n 17792dc168fd 2026-02-22T23:16:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/split-brain 2026-02-22T23:16:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Flagged image for resync from primary 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror split-brain 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+replaying 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:27.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:27.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-22T23:16:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T23:16:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='split-brain: 2026-02-22T23:16:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d16dd329-25e2-4da5-8acb-642656d49280 2026-02-22T23:16:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+error 2026-02-22T23:16:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: split-brain 2026-02-22T23:16:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:16:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:23 2026-02-22T23:16:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:16:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:16:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:16:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:16:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:16:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:16:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:16:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-22T23:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T23:16:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd: error opening image split-brain: (2) No such file or directory 2026-02-22T23:16:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status= 2026-02-22T23:16:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:16:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 1 2026-02-22T23:16:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:16:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-22T23:16:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='split-brain: 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: d16dd329-25e2-4da5-8acb-642656d49280 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802165,"replay_state":"idle"} 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:33 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:16:34.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:16:34.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:16:34.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:16:34.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802165,"replay_state":"idle"} 2026-02-22T23:16:34.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:16:34.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:16:34.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:16:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:16:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:16:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:16:34.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:16:34.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:16:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:33' 2026-02-22T23:16:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+replaying '2026-02-22 23:16:33' 2026-02-22T23:16:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:16:33' 2026-02-22T23:16:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:16:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING160046009mirror.0trial08219.2.3-689-gf90f4eaetrueOK60116016mirror.1trial08219.2.3-689-gf90f4eaefalseOK60186022mirror.2trial08219.2.3-689-gf90f4eaefalseOK60266032mirror.3trial08219.2.3-689-gf90f4eaefalseOKsplit-braind16dd329-25e2-4da5-8acb-642656d49280up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802165,"replay_state":"idle"}60046009mirror.0trial0822026-02-22 23:16:33cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:16:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:16:34.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T23:16:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:33' 2026-02-22T23:16:34.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T23:16:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:16:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:16:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:16:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:16:34.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror split-brain 2026-02-22T23:16:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:16:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror split-brain 2026-02-22T23:16:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=split-brain 2026-02-22T23:16:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/split-brain 2026-02-22T23:16:34.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/split-brain 2026-02-22T23:16:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T23:16:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-02-22T23:16:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:16:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:16:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:16:35 TEST: check if removed 'images'\''' OMAP are removed 2026-02-22T23:16:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:16:35 TEST: check if removed images' OMAP are removed 2026-02-22T23:16:35.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirrors cluster2 2026-02-22T23:16:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:16:35.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster2:0 2026-02-22T23:16:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:0 2026-02-22T23:16:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:16:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:16:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster2:0 2026-02-22T23:16:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:16:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:16:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster2 2026-02-22T23:16:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T23:16:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T23:16:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster2 2026-02-22T23:16:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:16:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T23:16:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:16:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:16:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster2:1 2026-02-22T23:16:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:1 2026-02-22T23:16:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:16:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster2:1 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster2 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster2 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:16:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster2:2 2026-02-22T23:16:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:2 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster2:2 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster2 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster2 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:16:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T23:16:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:16:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:16:35.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:35.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster2:3 2026-02-22T23:16:35.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:3 2026-02-22T23:16:35.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster2:3 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster2 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster2 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T23:16:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:16:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:16:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-22T23:16:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-22T23:16:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirroring 2026-02-22T23:16:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=status_global 2026-02-22T23:16:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T23:16:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T23:16:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-22T23:16:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T23:16:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=status_global 2026-02-22T23:16:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T23:16:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T23:16:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T23:16:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c status_global 2026-02-22T23:16:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T23:16:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T23:16:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T23:16:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T23:16:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-22T23:16:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirroring 2026-02-22T23:16:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=image_ 2026-02-22T23:16:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T23:16:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T23:16:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-22T23:16:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T23:16:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_ 2026-02-22T23:16:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T23:16:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T23:16:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T23:16:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_ 2026-02-22T23:16:35.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T23:16:35.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T23:16:35.211 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T23:16:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T23:16:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-22T23:16:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T23:16:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=image_map 2026-02-22T23:16:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T23:16:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T23:16:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T23:16:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T23:16:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_map 2026-02-22T23:16:35.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T23:16:35.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:35.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:35.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T23:16:35.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T23:16:35.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_map 2026-02-22T23:16:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 1 = 0 2026-02-22T23:16:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=1 2026-02-22T23:16:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T23:16:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T23:16:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T23:16:35.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_map 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_map 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T23:16:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T23:16:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 1 = 0 2026-02-22T23:16:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=1 2026-02-22T23:16:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T23:16:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T23:16:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T23:16:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:16:38.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T23:16:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T23:16:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T23:16:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_map 2026-02-22T23:16:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T23:16:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T23:16:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T23:16:38.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_map 2026-02-22T23:16:38.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirroring 2026-02-22T23:16:38.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=status_global 2026-02-22T23:16:38.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T23:16:38.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:38.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T23:16:38.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-22T23:16:38.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:38.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:38.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T23:16:38.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=status_global 2026-02-22T23:16:38.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T23:16:38.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:38.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T23:16:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T23:16:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c status_global 2026-02-22T23:16:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T23:16:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T23:16:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T23:16:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T23:16:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-22T23:16:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirroring 2026-02-22T23:16:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=image_ 2026-02-22T23:16:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T23:16:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T23:16:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-22T23:16:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T23:16:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_ 2026-02-22T23:16:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T23:16:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirroring 2026-02-22T23:16:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T23:16:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_ 2026-02-22T23:16:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T23:16:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local filter=image_map 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T23:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-22T23:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T23:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local filter=image_map 2026-02-22T23:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-22T23:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T23:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T23:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -c image_map 2026-02-22T23:16:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 0 = 0 2026-02-22T23:16:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ error_code=0 2026-02-22T23:16:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set -e 2026-02-22T23:16:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T23:16:38.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z '' ']' 2026-02-22T23:16:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: no blocklists' 2026-02-22T23:16:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:16:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:16:38.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:16:38 TEST: no blocklists 2026-02-22T23:16:38.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:16:38 TEST: no blocklists 2026-02-22T23:16:38.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS='--id admin' 2026-02-22T23:16:38.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster1 osd blocklist ls 2026-02-22T23:16:38.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -q 'listed 0 entries' 2026-02-22T23:16:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS='--id admin' 2026-02-22T23:16:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster2 osd blocklist ls 2026-02-22T23:16:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -q 'listed 0 entries' 2026-02-22T23:16:38.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:16:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z '' ']' 2026-02-22T23:16:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: test partially synced demote snapshot sync after daemon restart' 2026-02-22T23:16:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:16:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:16:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:16:38 TEST: test partially synced demote snapshot sync after daemon restart 2026-02-22T23:16:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:16:38 TEST: test partially synced demote snapshot sync after daemon restart 2026-02-22T23:16:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image=test_demote_image 2026-02-22T23:16:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test_demote_image snapshot 10G 2026-02-22T23:16:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:38.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:38.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:38.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:38.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:16:38.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:16:38.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:38.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test_demote_image 10G 2026-02-22T23:16:38.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:38.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:38.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:38.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:38.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:16:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 10G ']' 2026-02-22T23:16:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=10G 2026-02-22T23:16:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:16:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 10G --image-feature layering,exclusive-lock mirror/test_demote_image 2026-02-22T23:16:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:16:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:16:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test_demote_image snapshot 2026-02-22T23:16:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:16:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_demote_image snapshot 2026-02-22T23:16:39.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:16:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test_demote_image 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_demote_image': 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 10 GiB in 2560 objects 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 1846e2e8b18 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.1846e2e8b18 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:16:38 2026 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:16:38 2026 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:16:38 2026 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 78c5d576-ac42-4db9-a011-9357b9a1e671 2026-02-22T23:16:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:16:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-22T23:16:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:16:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:16:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:16:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:16:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:16:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:16:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:16:39.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_demote_image 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:39.131 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-22T23:16:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:16:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:16:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:16:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:16:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:16:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:16:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:16:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:16:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:16:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:16:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:16:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:16:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:16:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T23:16:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:16:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:16:40.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:16:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:16:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:16:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:16:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:16:40.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:16:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:16:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:16:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:16:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:16:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:16:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:16:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:16:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-02-22T23:16:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:2 2026-02-22T23:16:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:16:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:2 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:2 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T23:16:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:16:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:16:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:16:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:16:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T23:16:40.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:16:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:16:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:16:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:16:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:16:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:16:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:16:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_demote_image 2026-02-22T23:16:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:3 2026-02-22T23:16:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:16:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:3 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:3 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T23:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:16:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:16:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:16:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:16:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T23:16:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:16:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:16:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:16:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:16:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T23:16:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:16:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:16:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:16:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:16:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:16:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:16:41.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:16:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:16:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:16:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:16:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:16:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:16:41.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:16:41.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:16:41.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:16:41.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:16:41.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:16:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_demote_image 2026-02-22T23:16:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:41.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_demote_image started 2026-02-22T23:16:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:16:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:16:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_demote_image started 2026-02-22T23:16:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:16:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:16:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:16:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_demote_image 2026-02-22T23:16:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:16:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:16:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:16:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:16:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:16:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:16:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:16:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:16:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:16:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:16:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:16:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:16:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:16:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:16:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:16:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:16:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:16:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:16:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:16:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:16:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:16:42.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:16:42.488 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:16:42.488 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:16:42.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:16:42.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:16:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:16:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:16:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T23:16:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:16:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:16:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:16:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test_demote_image 2026-02-22T23:16:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_demote_image 2026-02-22T23:16:43.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 194 2026-02-22T23:16:43.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T23:16:43.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:16:43.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:43.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:43.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:43.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T23:16:43.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-22T23:16:43.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:16:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:16:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-22T23:16:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-22T23:16:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:16:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:16:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:16:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:43.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:16:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:16:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=194 2026-02-22T23:16:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:16:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:16:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:16:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:43.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:43.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:16:43.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:16:43.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=193 2026-02-22T23:16:43.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 194 = 193 2026-02-22T23:16:43.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:16:43.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:16:43.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:16:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:16:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=193 2026-02-22T23:16:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 194 = 193 2026-02-22T23:16:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:16:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T23:16:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:16:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:16:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=194 2026-02-22T23:16:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 194 = 194 2026-02-22T23:16:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+replaying 2026-02-22T23:16:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:44.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:44.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:45.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+replaying '' '' 2026-02-22T23:16:45.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:45.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:45.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T23:16:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:16:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 78c5d576-ac42-4db9-a011-9357b9a1e671 2026-02-22T23:16:45.636 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:16:45.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802199,"replay_state":"idle"} 2026-02-22T23:16:45.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:16:45.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:41 2026-02-22T23:16:45.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:16:45.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:16:45.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:16:45.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:16:45.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:45' 2026-02-22T23:16:45.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:16:45.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:16:45.638 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:16:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802199,"replay_state":"idle"} 2026-02-22T23:16:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:16:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:16:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:16:45.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:16:45.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:16:45.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:16:45.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:16:45.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:16:45.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:41' 2026-02-22T23:16:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+replaying '2026-02-22 23:16:41' 2026-02-22T23:16:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:16:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:16:41' 2026-02-22T23:16:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:16:45.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK160046009mirror.0trial08219.2.3-689-gf90f4eaetrueOK60116016mirror.1trial08219.2.3-689-gf90f4eaefalseOK60186022mirror.2trial08219.2.3-689-gf90f4eaefalseOK60266032mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_demote_image78c5d576-ac42-4db9-a011-9357b9a1e671up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802199,"replay_state":"idle"}60046009mirror.0trial0822026-02-22 23:16:41cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:16:45' 2026-02-22T23:16:45.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:16:45.676 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:16:45.677 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:41' 2026-02-22T23:16:45.677 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:16:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:16:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:16:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:16:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:16:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+stopped 2026-02-22T23:16:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:45.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:45.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:16:45.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:45.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:45.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:16:45.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:16:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+stopped '' '' 2026-02-22T23:16:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:16:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:16:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:16:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T23:16:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:16:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 78c5d576-ac42-4db9-a011-9357b9a1e671 2026-02-22T23:16:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:16:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:16:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:16:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:45 2026-02-22T23:16:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:16:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:16:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:16:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802199,"replay_state":"idle"} 2026-02-22T23:16:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:16:41 2026-02-22T23:16:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:16:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 193 .mirror.primary.78c5d576-ac42-4db9-a011-9357b9a1e671.dbdb9a1d-5a69-4744-9333-6d8a56f5d77b (peer_uuids:[]) 2026-02-22T23:16:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 194 .mirror.primary.78c5d576-ac42-4db9-a011-9357b9a1e671.ba64f0fd-01de-4240-932f-e939b07dd623 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:16:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:16:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:16:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:16:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:16:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:16:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:16:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:16:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:16:46.717 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:16:46.717 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:16:46.718 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:16:46.718 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:16:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:45' 2026-02-22T23:16:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+stopped '2026-02-22 23:16:45' 2026-02-22T23:16:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:16:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:16:45' 2026-02-22T23:16:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:16:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:16:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:16:46.748 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK161616164mirror.0trial08219.2.3-689-gf90f4eaetrueOK61686171mirror.1trial08219.2.3-689-gf90f4eaefalseOK61756182mirror.2trial08219.2.3-689-gf90f4eaefalseOK61846187mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_demote_image78c5d576-ac42-4db9-a011-9357b9a1e671up+stoppedlocal image is primary61616164mirror.0trial0822026-02-22 23:16:45cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802199,"replay_state":"idle"}2026-02-22 23:16:41' 2026-02-22T23:16:46.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:16:46.749 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:16:46.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:16:45' 2026-02-22T23:16:46.750 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:16:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:16:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:16:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:16:46.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:16:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_demote_image 2560 4194304 2026-02-22T23:16:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=2560 2026-02-22T23:16:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=4194304 2026-02-22T23:16:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 4194304 2026-02-22T23:16:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4194304 --io-threads 1 --io-total 10737418240 --io-pattern rand 2026-02-22T23:16:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4194304 io_threads 1 bytes 10737418240 pattern random 2026-02-22T23:16:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:16:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 1 412 417.598 1.6 GiB/s 2026-02-22T23:16:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 2 807 407.057 1.6 GiB/s 2026-02-22T23:16:49.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 3 1243 416.197 1.6 GiB/s 2026-02-22T23:16:50.784 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 4 1674 420.33 1.6 GiB/s 2026-02-22T23:16:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 5 2092 419.864 1.6 GiB/s 2026-02-22T23:16:52.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 6 2517 420.668 1.6 GiB/s 2026-02-22T23:16:52.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 6 ops: 2560 ops/sec: 418.647 bytes/sec: 1.6 GiB/s 2026-02-22T23:16:52.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ demote_image cluster2 mirror test_demote_image 2026-02-22T23:16:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_demote_image 2026-02-22T23:16:54.036 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image demoted to non-primary 2026-02-22T23:16:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_complete_mirror_snapshot_id cluster2 mirror test_demote_image primary_snap_id 2026-02-22T23:16:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local -n _snap_id=primary_snap_id 2026-02-22T23:16:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:54.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '(//snapshots/snapshot[namespace/complete='\''true'\'']/id)[last()]' 2026-02-22T23:16:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ _snap_id=195 2026-02-22T23:16:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_non_primary_snap_present cluster1 mirror test_demote_image 195 2026-02-22T23:16:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snap_id=195 2026-02-22T23:16:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:16:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T23:16:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.1 2026-02-22T23:16:54.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:54.178 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot/namespace[primary_snap_id='\''195'\'']' 2026-02-22T23:16:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:16:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirrors cluster1 -KILL 2026-02-22T23:16:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig=-KILL 2026-02-22T23:16:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:16:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:0 -KILL 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig=-KILL 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:0 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:16:54.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:16:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:16:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:16:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:16:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:16:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:16:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:16:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=82860 2026-02-22T23:16:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 82860 ']' 2026-02-22T23:16:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill -KILL 82860 2026-02-22T23:16:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:16:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:16:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=82860 '$2 == pid {print; exit 1}' 2026-02-22T23:16:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:16:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:16:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=82860 '$2 == pid {print; exit 1}' 2026-02-22T23:16:54.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:16:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:16:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:16:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:16:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:16:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:16:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:16:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:16:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:16:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:54.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:16:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T23:16:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:16:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:16:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:16:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:16:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:16:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:16:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:16:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:16:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:16:54.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:54.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:16:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:0 2026-02-22T23:16:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:16:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:16:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:16:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:16:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:16:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:16:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:16:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:16:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:16:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:16:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:1 -KILL 2026-02-22T23:16:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig=-KILL 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:1 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:16:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:16:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:16:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:16:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:16:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:16:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:16:54.322 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:16:54.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:16:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=82903 2026-02-22T23:16:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 82903 ']' 2026-02-22T23:16:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill -KILL 82903 2026-02-22T23:16:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:16:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:16:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=82903 '$2 == pid {print; exit 1}' 2026-02-22T23:16:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:16:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:16:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=82903 '$2 == pid {print; exit 1}' 2026-02-22T23:16:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:16:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:16:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:16:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:16:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:16:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:16:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:16:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:16:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:16:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:16:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:16:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:16:54.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:16:54.364 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:16:54.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T23:16:54.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:16:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:16:54.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:16:54.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:16:54.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:16:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:16:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:16:54.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:1 2026-02-22T23:16:54.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:16:54.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:16:54.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:16:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:16:54.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:16:54.390 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:16:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:2 -KILL 2026-02-22T23:16:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:16:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig=-KILL 2026-02-22T23:16:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:2 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:16:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:16:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:16:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:16:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:16:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:16:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:16:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:16:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:16:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=82965 2026-02-22T23:16:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 82965 ']' 2026-02-22T23:16:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill -KILL 82965 2026-02-22T23:16:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:16:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:16:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=82965 '$2 == pid {print; exit 1}' 2026-02-22T23:16:54.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:16:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:16:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=82965 '$2 == pid {print; exit 1}' 2026-02-22T23:16:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:16:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:16:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:16:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:16:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:16:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:16:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:16:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T23:16:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:16:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:16:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:16:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:16:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:16:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:16:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:16:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:16:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:16:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:16:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:16:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:16:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:16:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:2 2026-02-22T23:16:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:16:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:16:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:16:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:16:54.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:16:54.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:16:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:3 -KILL 2026-02-22T23:16:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:16:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig=-KILL 2026-02-22T23:16:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:16:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T23:16:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:3 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:16:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:16:54.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:16:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:16:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:16:54.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=83026 2026-02-22T23:16:54.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 83026 ']' 2026-02-22T23:16:54.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill -KILL 83026 2026-02-22T23:16:54.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:16:54.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:16:54.488 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:16:54.488 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=83026 '$2 == pid {print; exit 1}' 2026-02-22T23:16:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:16:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:16:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=83026 '$2 == pid {print; exit 1}' 2026-02-22T23:16:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:16:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:16:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:16:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:16:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:16:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:16:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:16:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:16:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:16:54.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:16:54.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:16:54.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:16:54.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:16:54.526 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:16:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T23:16:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:16:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:16:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:16:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:16:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:16:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:16:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:16:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:16:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:16:54.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:16:54.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:16:54.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:16:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:16:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:16:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:3 2026-02-22T23:16:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:16:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:16:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:16:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:16:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:16:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:16:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_snaps_json cluster1 mirror test_demote_image 2026-02-22T23:16:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:16:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:16:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_demote_image 2026-02-22T23:16:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 snap ls mirror/test_demote_image --all --format json 2026-02-22T23:16:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ SNAPS='[{"id":204,"name":".mirror.non_primary.78c5d576-ac42-4db9-a011-9357b9a1e671.9fb231b8-07c7-4368-bdd6-87e7f71d6eb5","size":10737418240,"protected":"false","timestamp":"Sun Feb 22 23:16:44 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":true,"primary_mirror_uuid":"6c842a54-ecb3-45b1-b677-4db9dda71c37","primary_snap_id":194,"last_copied_object_number":2560}},{"id":205,"name":".mirror.non_primary.78c5d576-ac42-4db9-a011-9357b9a1e671.fb50c391-7d33-4689-9e58-9dc8aa1752e9","size":10737418240,"protected":"false","timestamp":"Sun Feb 22 23:16:54 2026","namespace":{"type":"mirror","state":"demoted","mirror_peer_uuids":["70a926a6-5554-4bd7-a6ec-e3f48f4efa7b"],"complete":false,"primary_mirror_uuid":"6c842a54-ecb3-45b1-b677-4db9dda71c37","primary_snap_id":195,"last_copied_object_number":118}}]' 2026-02-22T23:16:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ jq -e '.[-1].namespace["type"] == "mirror" and .[-1].namespace["state"] == "demoted" and .[-1].namespace["complete"] == false' 2026-02-22T23:16:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:mirrordemoted70a926a6-5554-4bd7-a6ec-e3f48f4efa7bfalse6c842a54-ecb3-45b1-b677-4db9dda71c37195109true 2026-02-22T23:16:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirrors cluster1 2026-02-22T23:16:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:54.582 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:16:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:0 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T23:16:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:16:54.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T23:16:54.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:16:54.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:54.586 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:16:54.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:54.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:1 2026-02-22T23:16:54.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:16:54.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:16:54.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:16:54.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T23:16:54.602 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:16:54.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:16:54.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:16:54.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T23:16:54.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T23:16:54.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:16:54.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T23:16:54.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:16:54.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:54.603 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:16:54.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:54.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:2 2026-02-22T23:16:54.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:16:54.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:16:54.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:16:54.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T23:16:54.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:16:54.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:16:54.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:16:54.620 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T23:16:54.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T23:16:54.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:16:54.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T23:16:54.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:16:54.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:54.621 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:3 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:16:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:16:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T23:16:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:16:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:54.660 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:54.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:54.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T23:16:54.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-22T23:16:54.661 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:16:54.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:16:54.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-22T23:16:54.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-22T23:16:54.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:16:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:16:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:16:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.status 2026-02-22T23:16:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:54.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:16:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:16:54.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=195 2026-02-22T23:16:54.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:16:54.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:16:54.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:16:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:16:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:16:54.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:16:54.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:16:54.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:16:54.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:16:54.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:16:54.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:16:54.929 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.4 2026-02-22T23:16:55.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:55.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:55.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:55.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:55.330 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:55.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:55.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:16:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:16:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:16:55.359 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:16:55.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:16:55.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:16:55.360 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:16:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:16:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:16:55.361 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.8 2026-02-22T23:16:56.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:56.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:56.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:56.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:56.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:56.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:56.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:16:56.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:16:56.194 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:16:56.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:16:56.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:16:56.195 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:16:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:16:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:16:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:16:56.196 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1.6 2026-02-22T23:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:16:57.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:16:57.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:16:57.829 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:16:57.829 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:16:57.829 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:16:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:16:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:16:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:16:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:16:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:16:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:16:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:16:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:16:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:16:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:16:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:16:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:16:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:16:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:16:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:16:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:16:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:16:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:16:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:17:01.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:01.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:17:01.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:01.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:01.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:01.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:17:01.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:17:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:17:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:17:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:17:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:17:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:17:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:17:01.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:17:01.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:17:01.901 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:17:05.902 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:17:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:17:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:17:05.930 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:17:05.930 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:17:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:17:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:17:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:17:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:17:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:17:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:17:05.932 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 4 2026-02-22T23:17:09.933 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:17:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:17:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:17:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:17:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:17:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:17:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:17:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:17:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:17:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:17:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:17:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 8 2026-02-22T23:17:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:17:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:17.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:17.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:17:17.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:17:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:17:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:17:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:17:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:17:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:17:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:17:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:17:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:17:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 8 2026-02-22T23:17:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:17:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:17:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:17:26.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:17:26.239 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:17:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:17:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:17:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:17:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:17:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:17:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:17:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 16 2026-02-22T23:17:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:17:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:17:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:17:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:17:42.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:17:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:-:1.1: Document is empty 2026-02-22T23:17:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 2026-02-22T23:17:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:^ 2026-02-22T23:17:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id= 2026-02-22T23:17:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = '' 2026-02-22T23:17:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:17:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 16 2026-02-22T23:17:58.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:17:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.status 2026-02-22T23:17:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T23:17:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:17:58.309 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:17:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=195 2026-02-22T23:17:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 195 = 195 2026-02-22T23:17:58.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:17:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+unknown 2026-02-22T23:17:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:17:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:17:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:17:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:17:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:17:58.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:17:59.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+unknown '' '' 2026-02-22T23:17:59.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:17:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:17:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:17:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:17:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:17:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:17:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T23:17:59.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:17:59.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 78c5d576-ac42-4db9-a011-9357b9a1e671 2026-02-22T23:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:17:55 2026-02-22T23:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:17:54' 2026-02-22T23:17:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:17:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:17:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:17:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:17:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:17:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:17:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:17:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:17:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:17:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:17:59.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:17:59.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:17:59.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:17:55' 2026-02-22T23:17:59.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+unknown '2026-02-22 23:17:55' 2026-02-22T23:17:59.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:17:59.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:59.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:59.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:17:59.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:17:55' 2026-02-22T23:17:59.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:17:59.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:17:59.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:17:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING161616164mirror.0trial08219.2.3-689-gf90f4eaetrueOK61686171mirror.1trial08219.2.3-689-gf90f4eaefalseOK61756182mirror.2trial08219.2.3-689-gf90f4eaefalseOK61846187mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_demote_image78c5d576-ac42-4db9-a011-9357b9a1e671up+unknownremote image is not primary61616164mirror.0trial0822026-02-22 23:17:55cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+unknownremote image is not primary2026-02-22 23:17:54' 2026-02-22T23:17:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:17:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:17:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:17:55' 2026-02-22T23:17:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:17:59.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:17:59.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:17:59.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:17:59.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:17:59.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:17:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:17:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+unknown 2026-02-22T23:17:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:17:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:17:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:17:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:17:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:17:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:17:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:17:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:18:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-02-22T23:18:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:18:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:18:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:18:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:18:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:18:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T23:18:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:18:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 78c5d576-ac42-4db9-a011-9357b9a1e671 2026-02-22T23:18:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:18:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:18:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:18:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:17:59 2026-02-22T23:18:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:18:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:18:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+unknown 2026-02-22T23:18:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: remote image is not primary 2026-02-22T23:18:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:18:00' 2026-02-22T23:18:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T23:18:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+unknown 2026-02-22T23:18:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:18:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: remote image is not primary 2026-02-22T23:18:00.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:18:00.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:18:00.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:18:00.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:18:00.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:18:00.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:18:00.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:18:00.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:18:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:17:59' 2026-02-22T23:18:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+unknown '2026-02-22 23:17:59' 2026-02-22T23:18:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+unknown 2026-02-22T23:18:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:17:59' 2026-02-22T23:18:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:18:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:18:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:18:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING161676172mirror.0trial08219.2.3-689-gf90f4eaefalseOK61746179mirror.1trial08219.2.3-689-gf90f4eaetrueOK61816186mirror.2trial08219.2.3-689-gf90f4eaefalseOK61886194mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_demote_image78c5d576-ac42-4db9-a011-9357b9a1e671up+unknownremote image is not primary61676172mirror.0trial0822026-02-22 23:17:59cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+unknownremote image is not primary2026-02-22 23:18:00' 2026-02-22T23:18:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:18:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:18:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:17:59' 2026-02-22T23:18:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:18:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+unknown 2026-02-22T23:18:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+unknown 2026-02-22T23:18:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+unknown 2026-02-22T23:18:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+unknown 2026-02-22T23:18:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ compare_images mirror test_demote_image 2026-02-22T23:18:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local ret=0 2026-02-22T23:18:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_demote_image.export 2026-02-22T23:18:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_demote_image.export 2026-02-22T23:18:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:18:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local rmt_export=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.export 2026-02-22T23:18:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_demote_image.export 2026-02-22T23:18:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_demote_image.export 2026-02-22T23:18:00.460 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:18:00.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local loc_export=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.export 2026-02-22T23:18:00.461 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.export 2026-02-22T23:18:00.462 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 export mirror/test_demote_image /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.export 2026-02-22T23:18:05.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T23:18:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 export mirror/test_demote_image /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.export 2026-02-22T23:18:10.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-22T23:18:10.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cmp /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.export 2026-02-22T23:18:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_demote_image.export /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_demote_image.export 2026-02-22T23:18:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test_demote_image 2026-02-22T23:18:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force= 2026-02-22T23:18:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_demote_image 2026-02-22T23:18:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster2 mirror test_demote_image 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T23:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:18:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T23:18:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:18:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:18:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:18:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:18:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:18:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:18:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:18:18.095 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:18:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:18:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:18:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:18:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:18:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:18:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:18:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:18:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:18:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:18:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:18:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:18:18.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:18:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:18:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:18:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:18:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = stopped 2026-02-22T23:18:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 2 2026-02-22T23:18:20.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T23:18:20.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:20.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:20.157 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:18:20.158 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:20.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:18:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:18:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:18:20.162 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:18:20.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:18:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:18:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:18:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:18:20.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:18:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:18:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:18:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:18:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:18:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:18:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:18:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:18:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:18:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:18:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T23:18:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:18:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:18:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:18:20.222 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:18:20.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:18:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:18:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:18:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:20.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+stopped 2026-02-22T23:18:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:18:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:18:20.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:18:20.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:20.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:18:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+stopped '' '' 2026-02-22T23:18:21.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:18:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:18:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:18:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:18:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:18:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T23:18:21.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:18:21.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 78c5d576-ac42-4db9-a011-9357b9a1e671 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:18:19 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802297,"replay_state":"idle"} 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:18:20 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 206 .mirror.primary.78c5d576-ac42-4db9-a011-9357b9a1e671.fe12ca24-b05a-4046-a67a-425f07f98014 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:18:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:18:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:18:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:18:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:18:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:18:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:18:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:18:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:18:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:18:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:18:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:18:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:18:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:18:19' 2026-02-22T23:18:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+stopped '2026-02-22 23:18:19' 2026-02-22T23:18:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:18:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:18:19' 2026-02-22T23:18:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:18:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:18:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:18:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK161676172mirror.0trial08219.2.3-689-gf90f4eaefalseOK61746179mirror.1trial08219.2.3-689-gf90f4eaetrueOK61816186mirror.2trial08219.2.3-689-gf90f4eaefalseOK61886194mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_demote_image78c5d576-ac42-4db9-a011-9357b9a1e671up+stoppedlocal image is primary61676172mirror.0trial0822026-02-22 23:18:19cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802297,"replay_state":"idle"}2026-02-22 23:18:20' 2026-02-22T23:18:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:18:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:18:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:18:19' 2026-02-22T23:18:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:18:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:18:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:18:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:18:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:18:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+replaying 2026-02-22T23:18:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:18:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:18:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:18:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:18:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+replaying '' '' 2026-02-22T23:18:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:18:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:18:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:18:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:18:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:18:22.344 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T23:18:22.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_demote_image: 2026-02-22T23:18:22.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 78c5d576-ac42-4db9-a011-9357b9a1e671 2026-02-22T23:18:22.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:18:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802297,"replay_state":"idle"} 2026-02-22T23:18:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:18:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:18:20 2026-02-22T23:18:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:18:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:18:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:18:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:18:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:18:19' 2026-02-22T23:18:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:18:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:18:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:18:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802297,"replay_state":"idle"} 2026-02-22T23:18:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:18:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:18:22.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:18:22.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:18:22.403 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:18:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:18:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:18:22.404 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:18:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:18:20' 2026-02-22T23:18:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+replaying '2026-02-22 23:18:20' 2026-02-22T23:18:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:22.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:18:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:18:20' 2026-02-22T23:18:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:18:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:18:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:18:22.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK161616164mirror.0trial08219.2.3-689-gf90f4eaetrueOK61686171mirror.1trial08219.2.3-689-gf90f4eaefalseOK61756182mirror.2trial08219.2.3-689-gf90f4eaefalseOK61846187mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_demote_image78c5d576-ac42-4db9-a011-9357b9a1e671up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802297,"replay_state":"idle"}61616164mirror.0trial0822026-02-22 23:18:20cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:18:19' 2026-02-22T23:18:22.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:18:22.489 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T23:18:22.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:18:20' 2026-02-22T23:18:22.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T23:18:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:18:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:18:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:18:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:18:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster1 mirror test_demote_image 2026-02-22T23:18:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:18:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:18:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster1 mirror test_demote_image 2026-02-22T23:18:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_demote_image 2026-02-22T23:18:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_demote_image 2026-02-22T23:18:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 rm mirror/test_demote_image 2026-02-22T23:18:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T23:18:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:23.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ testlog 'TEST: force promote with a user snapshot' 2026-02-22T23:18:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ tee -a /tmp/tmp.VQMrvUqClS/rbd-mirror.test.log 2026-02-22T23:18:23.757 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ date '+%F %T' 2026-02-22T23:18:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo 2026-02-22 23:18:23 TEST: force promote with a user snapshot 2026-02-22T23:18:23.759 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:2026-02-22 23:18:23 TEST: force promote with a user snapshot 2026-02-22T23:18:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ force_promote_image=test_force_promote_user 2026-02-22T23:18:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image_and_enable_mirror cluster2 mirror test_force_promote_user snapshot 10G 2026-02-22T23:18:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:18:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:18:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:18:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:18:23.760 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n snapshot ']' 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_image cluster2 mirror test_force_promote_user 10G 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=128 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 10G ']' 2026-02-22T23:18:23.761 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=10G 2026-02-22T23:18:23.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ shift 2026-02-22T23:18:23.762 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 create --size 10G --image-feature layering,exclusive-lock mirror/test_force_promote_user 2026-02-22T23:18:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T23:18:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' image = image ']' 2026-02-22T23:18:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ enable_mirror cluster2 mirror test_force_promote_user snapshot 2026-02-22T23:18:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local mode=snapshot 2026-02-22T23:18:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_force_promote_user snapshot 2026-02-22T23:18:24.059 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Mirroring enabled 2026-02-22T23:18:24.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 info mirror/test_force_promote_user 2026-02-22T23:18:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_force_promote_user': 2026-02-22T23:18:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 10 GiB in 2560 objects 2026-02-22T23:18:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:18:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 1 2026-02-22T23:18:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 18893288a3f3 2026-02-22T23:18:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.18893288a3f3 2026-02-22T23:18:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:18:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock 2026-02-22T23:18:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:18:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:18:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:18:23 2026 2026-02-22T23:18:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:18:23 2026 2026-02-22T23:18:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:18:23 2026 2026-02-22T23:18:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:18:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:18:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 7a8eebb6-0564-455a-89f4-061f5addb2dd 2026-02-22T23:18:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: true 2026-02-22T23:18:24.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-22T23:18:24.092 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:18:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:18:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:18:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:18:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:18:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:18:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:18:24.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 12500.1 bytes/sec: 49 MiB/s 2026-02-22T23:18:24.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote_user 2026-02-22T23:18:24.133 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T23:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:24.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:18:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T23:18:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:18:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:18:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:18:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:18:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:18:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:18:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:18:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:18:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:25.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:18:25.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:18:25.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:18:25.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:25.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:25.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:18:25.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:25.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:25.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:18:25.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:18:25.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:25.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:25.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:18:25.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:18:25.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:18:25.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:18:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:18:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:18:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:18:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:18:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:25.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:18:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:18:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:18:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:18:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:18:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:18:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:18:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:18:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:18:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:18:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:18:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:18:25.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:25.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:25.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:18:25.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:18:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:18:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:25.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:18:25.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:18:25.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:18:25.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:18:25.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:25.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:18:25.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:18:25.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:18:25.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:18:25.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:18:25.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:25.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:18:25.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:18:25.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:18:25.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:25.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:25.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:18:25.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:18:25.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:18:25.209 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:18:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:18:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:18:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:18:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:18:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:18:25.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:18:25.215 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:18:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:18:25.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:18:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:2 2026-02-22T23:18:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:2 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:2 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:18:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T23:18:25.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:18:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:18:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:18:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:25.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:18:25.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:18:25.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:18:25.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:25.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:25.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:3 2026-02-22T23:18:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:18:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:18:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:18:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:18:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:18:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:18:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:18:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:18:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T23:18:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:3 2026-02-22T23:18:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:18:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T23:18:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:3 2026-02-22T23:18:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:18:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:18:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T23:18:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:18:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:18:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:18:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:18:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T23:18:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:18:25.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:18:25.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:18:25.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:18:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:18:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:18:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:18:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:18:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:18:26.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:26.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:26.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:26.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:18:26.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:18:26.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:18:26.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:18:26.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:18:26.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:18:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:18:26.388 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:26.441 INFO:tasks.ceph.cluster2.osd.0.trial082.stderr:2026-02-22T23:18:26.439+0000 7f0537685640 -1 reset not still connected to 0x55fd4ed69380 2026-02-22T23:18:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:18:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:18:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:18:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:18:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:18:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:18:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:18:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:18:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:18:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:18:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:18:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:18:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:18:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:18:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:26.531 INFO:tasks.ceph.cluster2.osd.2.trial082.stderr:2026-02-22T23:18:26.529+0000 7f6372547640 -1 reset not still connected to 0x557ed58320d0 2026-02-22T23:18:26.531 INFO:tasks.ceph.cluster2.osd.2.trial082.stderr:2026-02-22T23:18:26.529+0000 7f6372547640 -1 reset not still connected to 0x557ed72b0a90 2026-02-22T23:18:26.536 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:18:26.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:18:26.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:18:26.537 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:18:26.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:18:26.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:26.539 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_started cluster1 mirror test_force_promote_user 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote_user started 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=started 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:18:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_force_promote_user started 2026-02-22T23:18:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=started 2026-02-22T23:18:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:18:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:18:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:18:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:18:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:18:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:18:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:18:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:18:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:18:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:18:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:18:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:18:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:18:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:18:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:18:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:18:27.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:18:27.559 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:18:27.559 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:18:27.559 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:18:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:18:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:18:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:18:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:18:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:18:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:18:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:18:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:18:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:18:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:18:27.616 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:18:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:18:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:18:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:18:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:18:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T23:18:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Replaying"' 2026-02-22T23:18:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:18:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: "state": "Replaying" 2026-02-22T23:18:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ current_state=started 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test started = started 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_force_promote_user 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote_user 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote_user 2026-02-22T23:18:28.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 198 2026-02-22T23:18:28.067 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_force_promote_user 2026-02-22T23:18:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_cluster=cluster1 2026-02-22T23:18:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster2-mirror-test_force_promote_user.status 2026-02-22T23:18:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster2-mirror-test_force_promote_user.status 2026-02-22T23:18:28.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:18:28.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_force_promote_user.status 2026-02-22T23:18:28.069 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ mkfname cluster1-mirror-test_force_promote_user.status 2026-02-22T23:18:28.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo cluster1-mirror-test_force_promote_user.status 2026-02-22T23:18:28.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T23:18:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local local_status_log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_force_promote_user.status 2026-02-22T23:18:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_force_promote_user /tmp/tmp.VQMrvUqClS/cluster2-mirror-test_force_promote_user.status 2026-02-22T23:18:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster2-mirror-test_force_promote_user.status 2026-02-22T23:18:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_force_promote_user --format xml 2026-02-22T23:18:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:18:28.100 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T23:18:28.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snapshot_id=198 2026-02-22T23:18:28.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ true 2026-02-22T23:18:28.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T23:18:28.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0.2 2026-02-22T23:18:28.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote_user /tmp/tmp.VQMrvUqClS/cluster1-mirror-test_force_promote_user.status 2026-02-22T23:18:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local log=/tmp/tmp.VQMrvUqClS/cluster1-mirror-test_force_promote_user.status 2026-02-22T23:18:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote_user --format xml 2026-02-22T23:18:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T23:18:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T23:18:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local primary_snapshot_id=198 2026-02-22T23:18:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test 198 = 198 2026-02-22T23:18:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_force_promote_user 2026-02-22T23:18:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = journal ']' 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote_user up+replaying 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:18:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote_user up+replaying '' '' 2026-02-22T23:18:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:18:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:18:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:18:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:18:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:18:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote_user 2026-02-22T23:18:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_force_promote_user: 2026-02-22T23:18:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 7a8eebb6-0564-455a-89f4-061f5addb2dd 2026-02-22T23:18:29.369 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:18:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802304,"replay_state":"idle"} 2026-02-22T23:18:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:18:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:18:25 2026-02-22T23:18:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:18:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:18:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: down+unknown 2026-02-22T23:18:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: status not found 2026-02-22T23:18:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: ' 2026-02-22T23:18:29.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T23:18:29.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+replaying 2026-02-22T23:18:29.371 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:18:29.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802304,"replay_state":"idle"} 2026-02-22T23:18:29.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:18:29.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:18:29.372 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:18:29.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:18:29.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:18:29.373 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:18:29.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:18:29.374 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:18:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:18:25' 2026-02-22T23:18:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote_user up+replaying '2026-02-22 23:18:25' 2026-02-22T23:18:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+replaying 2026-02-22T23:18:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:18:25' 2026-02-22T23:18:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:18:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:18:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:18:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='WARNINGOKWARNING161676172mirror.0trial08219.2.3-689-gf90f4eaefalseOK61746179mirror.1trial08219.2.3-689-gf90f4eaetrueOK61816186mirror.2trial08219.2.3-689-gf90f4eaefalseOK61886194mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_force_promote_user7a8eebb6-0564-455a-89f4-061f5addb2ddup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802304,"replay_state":"idle"}61746179mirror.1trial0822026-02-22 23:18:25cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37down+unknownstatus not found' 2026-02-22T23:18:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:18:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T23:18:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:18:25' 2026-02-22T23:18:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T23:18:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+replaying 2026-02-22T23:18:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+replaying 2026-02-22T23:18:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+replaying 2026-02-22T23:18:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+replaying 2026-02-22T23:18:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped 2026-02-22T23:18:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:18:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:18:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:18:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:18:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:18:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped '' '' 2026-02-22T23:18:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:18:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:18:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:18:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:18:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:18:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote_user 2026-02-22T23:18:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_force_promote_user: 2026-02-22T23:18:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 7a8eebb6-0564-455a-89f4-061f5addb2dd 2026-02-22T23:18:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:18:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:18:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:18:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:18:30 2026-02-22T23:18:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:18:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:18:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+replaying 2026-02-22T23:18:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802304,"replay_state":"idle"} 2026-02-22T23:18:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:18:25 2026-02-22T23:18:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:18:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 197 .mirror.primary.7a8eebb6-0564-455a-89f4-061f5addb2dd.9ca3de85-e5ff-4028-aa65-6799143c1803 (peer_uuids:[]) 2026-02-22T23:18:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 198 .mirror.primary.7a8eebb6-0564-455a-89f4-061f5addb2dd.9d6c5e93-2e1c-4a9a-891e-8114644e46cd (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:18:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:18:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:18:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:18:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:18:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:18:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:18:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:18:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:18:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:18:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:18:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:18:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:18:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:18:30' 2026-02-22T23:18:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote_user up+stopped '2026-02-22 23:18:30' 2026-02-22T23:18:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:18:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:18:30' 2026-02-22T23:18:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:18:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:18:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:18:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK161616164mirror.0trial08219.2.3-689-gf90f4eaetrueOK61686171mirror.1trial08219.2.3-689-gf90f4eaefalseOK61756182mirror.2trial08219.2.3-689-gf90f4eaefalseOK61846187mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_force_promote_user7a8eebb6-0564-455a-89f4-061f5addb2ddup+stoppedlocal image is primary61686171mirror.1trial0822026-02-22 23:18:30cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771802304,"replay_state":"idle"}2026-02-22 23:18:25' 2026-02-22T23:18:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:18:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T23:18:30.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:18:30' 2026-02-22T23:18:30.481 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T23:18:30.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:18:30.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:18:30.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:18:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:18:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-22T23:18:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:18:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:18:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:18:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:18:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:18:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:18:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:18:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 10000.1 bytes/sec: 39 MiB/s 2026-02-22T23:18:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ create_snapshot cluster2 mirror test_force_promote_user snap1 2026-02-22T23:18:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap=snap1 2026-02-22T23:18:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 snap create mirror/test_force_promote_user@snap1 2026-02-22T23:18:31.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T23:18:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_force_promote_user 2560 4194304 2026-02-22T23:18:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=2560 2026-02-22T23:18:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size=4194304 2026-02-22T23:18:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n 4194304 2026-02-22T23:18:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4194304 --io-threads 1 --io-total 10737418240 --io-pattern rand 2026-02-22T23:18:31.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4194304 io_threads 1 bytes 10737418240 pattern random 2026-02-22T23:18:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:18:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 1 371 373.873 1.5 GiB/s 2026-02-22T23:18:33.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 2 726 365.698 1.4 GiB/s 2026-02-22T23:18:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 3 1086 362.942 1.4 GiB/s 2026-02-22T23:18:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 4 1452 364.347 1.4 GiB/s 2026-02-22T23:18:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 5 1815 364.078 1.4 GiB/s 2026-02-22T23:18:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 6 2171 360.22 1.4 GiB/s 2026-02-22T23:18:38.111 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: 7 2532 360.483 1.4 GiB/s 2026-02-22T23:18:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 7 ops: 2560 ops/sec: 360.516 bytes/sec: 1.4 GiB/s 2026-02-22T23:18:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:18:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote_user 2026-02-22T23:18:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:18:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote_user 2026-02-22T23:18:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Snapshot ID: 200 2026-02-22T23:18:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_snap_present cluster1 mirror test_force_promote_user snap1 2026-02-22T23:18:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local snap_name=snap1 2026-02-22T23:18:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:18:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T23:18:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:18:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 info mirror/test_force_promote_user@snap1 2026-02-22T23:18:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:rbd image 'test_force_promote_user': 2026-02-22T23:18:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: size 10 GiB in 2560 objects 2026-02-22T23:18:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: order 22 (4 MiB objects) 2026-02-22T23:18:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: snapshot_count: 3 2026-02-22T23:18:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: id: 18233ab764ae 2026-02-22T23:18:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: block_name_prefix: rbd_data.18233ab764ae 2026-02-22T23:18:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: format: 2 2026-02-22T23:18:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: features: layering, exclusive-lock, non-primary 2026-02-22T23:18:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: op_features: 2026-02-22T23:18:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: flags: 2026-02-22T23:18:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: create_timestamp: Sun Feb 22 23:18:25 2026 2026-02-22T23:18:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: access_timestamp: Sun Feb 22 23:18:25 2026 2026-02-22T23:18:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: modify_timestamp: Sun Feb 22 23:18:25 2026 2026-02-22T23:18:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: protected: False 2026-02-22T23:18:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring state: enabled 2026-02-22T23:18:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring mode: snapshot 2026-02-22T23:18:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring global id: 7a8eebb6-0564-455a-89f4-061f5addb2dd 2026-02-22T23:18:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: mirroring primary: false 2026-02-22T23:18:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:18:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:18:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirrors cluster1 -KILL 2026-02-22T23:18:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig=-KILL 2026-02-22T23:18:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:18:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:0 -KILL 2026-02-22T23:18:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:18:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig=-KILL 2026-02-22T23:18:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:18:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:18:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T23:18:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:0 2026-02-22T23:18:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:18:42.146 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:18:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:18:42.161 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:18:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=85432 2026-02-22T23:18:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 85432 ']' 2026-02-22T23:18:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill -KILL 85432 2026-02-22T23:18:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:18:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:18:42.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:18:42.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=85432 '$2 == pid {print; exit 1}' 2026-02-22T23:18:42.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:18:42.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:18:42.184 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=85432 '$2 == pid {print; exit 1}' 2026-02-22T23:18:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:18:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:18:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:18:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:18:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:18:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:18:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:18:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:18:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:18:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:18:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:18:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:18:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:18:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:18:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T23:18:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:18:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:18:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:18:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:18:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:18:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:18:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:18:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:18:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:18:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:18:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:42.221 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:18:42.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:18:42.234 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:18:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:0 2026-02-22T23:18:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:18:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:18:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:18:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:18:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:18:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:18:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:18:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:18:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:18:42.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:18:42.251 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:18:42.253 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:42.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:1 -KILL 2026-02-22T23:18:42.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:18:42.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig=-KILL 2026-02-22T23:18:42.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:18:42.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:18:42.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T23:18:42.254 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:1 2026-02-22T23:18:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:18:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:18:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:18:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:18:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:18:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:18:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:18:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:18:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:18:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:18:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:18:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:18:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=85467 2026-02-22T23:18:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 85467 ']' 2026-02-22T23:18:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill -KILL 85467 2026-02-22T23:18:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:18:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:18:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:18:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=85467 '$2 == pid {print; exit 1}' 2026-02-22T23:18:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:18:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:18:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=85467 '$2 == pid {print; exit 1}' 2026-02-22T23:18:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:18:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:18:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:18:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:18:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:18:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:18:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:18:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T23:18:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:18:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:18:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:18:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:18:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:18:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:18:42.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:18:42.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:18:42.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:18:42.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:18:42.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:18:42.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:18:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:18:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:18:42.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:1 2026-02-22T23:18:42.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:18:42.325 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:18:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:18:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:18:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:18:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:2 -KILL 2026-02-22T23:18:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:18:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig=-KILL 2026-02-22T23:18:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:18:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:2 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:18:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:18:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:18:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:18:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:18:42.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:18:42.349 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:18:42.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=85536 2026-02-22T23:18:42.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 85536 ']' 2026-02-22T23:18:42.350 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill -KILL 85536 2026-02-22T23:18:42.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:18:42.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:18:42.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:18:42.351 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=85536 '$2 == pid {print; exit 1}' 2026-02-22T23:18:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:18:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:18:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=85536 '$2 == pid {print; exit 1}' 2026-02-22T23:18:42.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:18:42.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:18:42.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:18:42.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:18:42.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:18:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:18:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:18:42.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T23:18:42.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:18:42.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:18:42.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.394 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:18:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:18:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:18:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:2 2026-02-22T23:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:18:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:18:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:18:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:18:42.420 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:18:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:3 -KILL 2026-02-22T23:18:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:18:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig=-KILL 2026-02-22T23:18:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:18:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:18:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T23:18:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:3 2026-02-22T23:18:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:18:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:18:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:18:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:18:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:18:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:18:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:18:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:18:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:18:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:18:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:18:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:18:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:18:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:18:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=85598 2026-02-22T23:18:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 85598 ']' 2026-02-22T23:18:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill -KILL 85598 2026-02-22T23:18:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:18:42.434 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:18:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:18:42.435 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=85598 '$2 == pid {print; exit 1}' 2026-02-22T23:18:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:18:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:18:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=85598 '$2 == pid {print; exit 1}' 2026-02-22T23:18:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:18:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:18:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:18:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:18:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:18:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:18:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:18:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:18:42.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T23:18:42.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:18:42.476 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:18:42.477 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:18:42.478 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:18:42.487 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:18:42.488 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:18:42.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:3 2026-02-22T23:18:42.490 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:18:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:18:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:18:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:18:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:18:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:18:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:18:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:18:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:18:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:18:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:18:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:18:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:18:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:18:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:18:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:18:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T23:18:42.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ get_snaps_json cluster1 mirror test_force_promote_user 2026-02-22T23:18:42.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:18:42.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local pool=mirror 2026-02-22T23:18:42.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local image=test_force_promote_user 2026-02-22T23:18:42.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 snap ls mirror/test_force_promote_user --all --format json 2026-02-22T23:18:42.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ SNAPS='[{"id":208,"name":".mirror.non_primary.7a8eebb6-0564-455a-89f4-061f5addb2dd.216364d9-65de-4b6c-a809-e88290bd47d8","size":10737418240,"protected":"false","timestamp":"Sun Feb 22 23:18:28 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":true,"primary_mirror_uuid":"6c842a54-ecb3-45b1-b677-4db9dda71c37","primary_snap_id":198,"last_copied_object_number":2560}},{"id":209,"name":"snap1","size":10737418240,"protected":"false","timestamp":"Sun Feb 22 23:18:40 2026","namespace":{"type":"user"}},{"id":210,"name":".mirror.non_primary.7a8eebb6-0564-455a-89f4-061f5addb2dd.0a321ad8-1122-4354-8153-862354951253","size":10737418240,"protected":"false","timestamp":"Sun Feb 22 23:18:41 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":false,"primary_mirror_uuid":"6c842a54-ecb3-45b1-b677-4db9dda71c37","primary_snap_id":200,"last_copied_object_number":919}}]' 2026-02-22T23:18:42.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ jq -e '.[-1].namespace["type"] == "mirror" and .[-1].namespace["state"] == "non-primary" and .[-1].namespace["complete"] == false' 2026-02-22T23:18:42.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:true 2026-02-22T23:18:42.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ promote_image cluster1 mirror test_force_promote_user --force 2026-02-22T23:18:42.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:18:42.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:18:42.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:18:42.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local force=--force 2026-02-22T23:18:42.533 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_force_promote_user --force 2026-02-22T23:19:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:Image promoted to primary 2026-02-22T23:19:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirrors cluster1 2026-02-22T23:19:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:19:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:19:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:0 2026-02-22T23:19:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:19:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:19:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:19:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:0 2026-02-22T23:19:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:19:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:19:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:19:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=0 2026-02-22T23:19:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:19:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=0 2026-02-22T23:19:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:19:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:19:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:19:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:1 2026-02-22T23:19:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:19:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:19:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:19:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:1 2026-02-22T23:19:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:19:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:19:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:19:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=1 2026-02-22T23:19:18.102 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:19:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=1 2026-02-22T23:19:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:19:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:19:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:19:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:2 2026-02-22T23:19:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:19:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:19:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:19:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:2 2026-02-22T23:19:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:19:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:19:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:19:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=2 2026-02-22T23:19:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:19:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=2 2026-02-22T23:19:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:19:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:19:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:19:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ start_mirror cluster1:3 2026-02-22T23:19:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:19:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance 2026-02-22T23:19:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:19:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local val=cluster1:3 2026-02-22T23:19:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster_var_name=cluster 2026-02-22T23:19:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local instance_var_name=instance 2026-02-22T23:19:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cluster=cluster1 2026-02-22T23:19:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ instance=3 2026-02-22T23:19:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval cluster=cluster1 2026-02-22T23:19:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ eval instance=3 2026-02-22T23:19:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:19:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:19:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T23:19:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster1 mirror test_force_promote_user 2026-02-22T23:19:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:19:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:19:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:19:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote_user stopped 2026-02-22T23:19:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:19:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:19:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:19:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:19:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:19:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:19:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:19:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster1 mirror test_force_promote_user stopped 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster1 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:19:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster1 2026-02-22T23:19:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:19:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:19:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:19:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:19:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:19:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:19:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:19:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:19:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:19:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:19:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:19:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:19:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:19:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:19:19.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:19.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:19:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:19:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:19:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:19:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:19:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:19:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:19:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:19:19.334 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:19:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:19:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 1 2026-02-22T23:19:20.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:19:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:19:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:19:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:19:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:19:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:20.439 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:20.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:20.440 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:19:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:20.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:19:20.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:19:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:19:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:19:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:20.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:20.495 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:19:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:19:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:19:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:19:20.500 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:19:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:19:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:19:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:20.550 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:19:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:19:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:19:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:20.553 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:19:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:19:20.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:20.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:20.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:20.563 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:20.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:20.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:20.604 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:20.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:19:20.605 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 2 2026-02-22T23:19:22.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:19:22.606 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:19:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:19:22.610 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:19:22.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:19:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:19:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:19:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:19:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:22.679 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:22.723 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:19:22.724 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:22.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:19:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:19:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:19:22.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:22.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:22.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:22.733 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:19:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:19:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:19:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:22.776 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:19:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:19:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:19:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:19:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:19:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:19:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:19:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:19:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:19:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:22.828 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:19:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 4 2026-02-22T23:19:26.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:19:26.830 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:19:26.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:26.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:26.832 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:19:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:19:26.834 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:19:26.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:26.846 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:26.847 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:19:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:19:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:19:26.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:26.903 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:26.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:26.904 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:19:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:19:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:19:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:19:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:19:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:19:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:19:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:19:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:19:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:19:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:19:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:19:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:26.960 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:19:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:19:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:19:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:19:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 8 2026-02-22T23:19:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:19:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:19:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:19:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:19:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:19:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:19:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:19:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:19:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:19:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:19:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:19:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:19:35.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:35.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:35.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:35.134 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:19:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:19:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:19:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:19:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:19:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:19:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:19:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:19:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:35.235 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:19:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:19:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:19:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:35.289 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:19:35.290 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 8 2026-02-22T23:19:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:19:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:19:43.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:43.294 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:43.295 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:43.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:43.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:43.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:43.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:19:43.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:19:43.297 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:19:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:19:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:19:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:19:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:43.353 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:19:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:19:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:19:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:19:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:19:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:19:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:19:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:19:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:19:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:19:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:43.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:43.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:43.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:43.463 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:19:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:19:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:19:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:19:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:19:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:19:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:19:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:19:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:19:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:19:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:19:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:19:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:19:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 8 2026-02-22T23:19:51.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:19:51.515 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:19:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:19:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:19:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:19:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:51.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:51.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:51.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:19:51.578 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:51.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:51.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:19:51.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:19:51.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:51.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:51.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:51.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:19:51.579 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:1 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:19:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:19:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:19:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:19:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:19:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:19:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:51.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:51.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:51.629 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:51.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:51.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:51.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:19:51.630 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:19:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:2 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:19:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:19:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:19:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:51.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:51.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:51.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:19:51.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:3 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:19:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:19:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:19:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:51.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:19:51.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:19:51.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:19:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 8 2026-02-22T23:19:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T23:19:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:19:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:19:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:19:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:19:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:19:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:19:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:19:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:19:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster1:0 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster1 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster1 2026-02-22T23:19:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:19:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:19:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:19:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:19:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:19:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:19:59.803 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:19:59.803 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:19:59.803 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:19:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:19:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:19:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote_user 2026-02-22T23:19:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:19:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:19:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:19:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T23:19:59.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:19:59.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:19:59.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:19:59.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state=stopped 2026-02-22T23:19:59.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local s 2026-02-22T23:19:59.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:19:59.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T23:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local test_state=stopped 2026-02-22T23:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status_result 2026-02-22T23:20:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local current_state=stopped 2026-02-22T23:20:00.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:20:00.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_instance=cluster2 2026-02-22T23:20:00.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ grep -i state 2026-02-22T23:20:00.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:20:00.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:20:00.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance=cluster2 2026-02-22T23:20:00.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local loop_instance 2026-02-22T23:20:00.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:20:00.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sleep 0 2026-02-22T23:20:00.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T23:20:00.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ seq 0 3 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:20:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:0 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:20:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:20:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:20:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:20:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:20:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:20:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T23:20:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T23:20:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:admin_socket: invalid command 2026-02-22T23:20:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:20:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T23:20:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:20:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ shift 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local local_cluster=cluster2:1 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:00.868 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:20:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:20:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval local_cluster=cluster2 2026-02-22T23:20:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ local_cluster=cluster2 2026-02-22T23:20:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:20:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:20:00.869 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:20:00.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:20:00.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:20:00.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:20:00.879 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T23:20:00.923 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ return 0 2026-02-22T23:20:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T23:20:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo ' "state": "Stopped"' 2026-02-22T23:20:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep -i Replaying 2026-02-22T23:20:00.925 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test stopped = stopped 2026-02-22T23:20:00.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:20:00.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote_user up+stopped 2026-02-22T23:20:00.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:20:00.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:00.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:00.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:20:00.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:20:00.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:20:00.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:20:00.926 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:20:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote_user up+stopped '' '' 2026-02-22T23:20:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:20:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:20:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:20:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:20:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:20:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:20:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote_user 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_force_promote_user: 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 7a8eebb6-0564-455a-89f4-061f5addb2dd 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.0 on trial082 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:19:58 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster2-DNE 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:20:00 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 212 .mirror.primary.7a8eebb6-0564-455a-89f4-061f5addb2dd.14009d38-81d5-40b8-ae1d-fa8f3319a797 (peer_uuids:[70a926a6-5554-4bd7-a6ec-e3f48f4efa7b])' 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:20:01.962 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:20:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:20:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:20:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:20:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.0 on trial082 2026-02-22T23:20:01.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:20:01.966 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:20:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:19:58' 2026-02-22T23:20:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote_user up+stopped '2026-02-22 23:19:58' 2026-02-22T23:20:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:20:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:20:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:19:58' 2026-02-22T23:20:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:20:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:20:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T23:20:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK162756280mirror.0trial08219.2.3-689-gf90f4eaetrueOK62796283mirror.1trial08219.2.3-689-gf90f4eaefalseOK62876291mirror.2trial08219.2.3-689-gf90f4eaefalseOK62936298mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_force_promote_user7a8eebb6-0564-455a-89f4-061f5addb2ddup+stoppedlocal image is primary62756280mirror.0trial0822026-02-22 23:19:58cluster2-DNE6c842a54-ecb3-45b1-b677-4db9dda71c37up+stoppedlocal image is primary2026-02-22 23:20:00' 2026-02-22T23:20:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:20:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T23:20:02.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:19:58' 2026-02-22T23:20:02.002 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T23:20:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:20:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:20:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:20:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:20:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:20:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:20:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped 2026-02-22T23:20:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:20:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:20:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:20:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:20:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T23:20:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 1 2026-02-22T23:20:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped '' '' 2026-02-22T23:20:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:20:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:20:03.008 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local description_pattern= 2026-02-22T23:20:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local service_pattern= 2026-02-22T23:20:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:20:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:20:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote_user 2026-02-22T23:20:03.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='test_force_promote_user: 2026-02-22T23:20:03.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: global_id: 7a8eebb6-0564-455a-89f4-061f5addb2dd 2026-02-22T23:20:03.038 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:20:03.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:20:03.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: service: mirror.1 on trial082 2026-02-22T23:20:03.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:20:00 2026-02-22T23:20:03.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: peer_sites: 2026-02-22T23:20:03.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: name: cluster1-DNE 2026-02-22T23:20:03.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: state: up+stopped 2026-02-22T23:20:03.039 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: description: local image is primary 2026-02-22T23:20:03.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: last_update: 2026-02-22 23:19:58 2026-02-22T23:20:03.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: snapshots: 2026-02-22T23:20:03.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 198 .mirror.primary.7a8eebb6-0564-455a-89f4-061f5addb2dd.9d6c5e93-2e1c-4a9a-891e-8114644e46cd (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae]) 2026-02-22T23:20:03.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: 200 .mirror.primary.7a8eebb6-0564-455a-89f4-061f5addb2dd.142ba486-f4c7-4129-a4b5-ee6fa17b58c4 (peer_uuids:[6a3341eb-d62f-469b-8fe4-d32d7815b1ae])' 2026-02-22T23:20:03.040 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T23:20:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: state: up+stopped 2026-02-22T23:20:03.041 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^ description: .*' 2026-02-22T23:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: description: local image is primary 2026-02-22T23:20:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n '' ']' 2026-02-22T23:20:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:20:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep '^up+' 2026-02-22T23:20:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:20:03.043 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T23:20:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: service: mirror.1 on trial082 2026-02-22T23:20:03.045 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update 2026-02-22T23:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T23:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:20:00' 2026-02-22T23:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote_user up+stopped '2026-02-22 23:20:00' 2026-02-22T23:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local state_pattern=up+stopped 2026-02-22T23:20:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local 'prev_last_update=2026-02-22 23:20:00' 2026-02-22T23:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local status 2026-02-22T23:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ CEPH_ARGS= 2026-02-22T23:20:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T23:20:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ status='OKOKOK161616164mirror.0trial08219.2.3-689-gf90f4eaetrueOK61686171mirror.1trial08219.2.3-689-gf90f4eaefalseOK61756182mirror.2trial08219.2.3-689-gf90f4eaefalseOK61846187mirror.3trial08219.2.3-689-gf90f4eaefalseOKtest_force_promote_user7a8eebb6-0564-455a-89f4-061f5addb2ddup+stoppedlocal image is primary61686171mirror.1trial0822026-02-22 23:20:00cluster1-DNE417b0160-ec0d-4455-98c9-aaede082a112up+stoppedlocal image is primary2026-02-22 23:19:58' 2026-02-22T23:20:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local last_update state 2026-02-22T23:20:03.074 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T23:20:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ last_update='2026-02-22 23:20:00' 2026-02-22T23:20:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T23:20:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ state=up+stopped 2026-02-22T23:20:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo up+stopped 2026-02-22T23:20:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ grep up+stopped 2026-02-22T23:20:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:up+stopped 2026-02-22T23:20:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:20:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:20:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster1 mirror test_force_promote_user 100 2026-02-22T23:20:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:20:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:20:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:20:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:20:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:20:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster1 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:20:03.311 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:20:03.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:20:03.319 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 12500.1 bytes/sec: 49 MiB/s 2026-02-22T23:20:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-22T23:20:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:20:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local count=100 2026-02-22T23:20:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local size= 2026-02-22T23:20:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:20:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ size=4096 2026-02-22T23:20:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T23:20:03.352 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T23:20:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T23:20:03.378 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:elapsed: 0 ops: 100 ops/sec: 3846.19 bytes/sec: 15 MiB/s 2026-02-22T23:20:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster1 mirror test_force_promote_user 2026-02-22T23:20:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:20:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:20:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:20:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster1 mirror test_force_promote_user 2026-02-22T23:20:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:20:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_force_promote_user 2026-02-22T23:20:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T23:20:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster1 rm mirror/test_force_promote_user 2026-02-22T23:20:05.299 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T23:20:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:20:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image_retry cluster2 mirror test_force_promote_user 2026-02-22T23:20:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:20:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:20:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:20:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ remove_image cluster2 mirror test_force_promote_user 2026-02-22T23:20:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:20:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pool=mirror 2026-02-22T23:20:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local image=test_force_promote_user 2026-02-22T23:20:05.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote_user 2026-02-22T23:20:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T23:20:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote_user 2026-02-22T23:20:07.689 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T23:20:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ return 0 2026-02-22T23:20:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ cleanup 0 2026-02-22T23:20:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local error_code=0 2026-02-22T23:20:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ set +e 2026-02-22T23:20:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -ne 0 ']' 2026-02-22T23:20:07.692 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z '' ']' 2026-02-22T23:20:07.693 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster instance 2026-02-22T23:20:07.693 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T23:20:07.693 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T23:20:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:pool 'mirror' does not exist 2026-02-22T23:20:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T23:20:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T23:20:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:pool 'mirror' does not exist 2026-02-22T23:20:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T23:20:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T23:20:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:pool 'mirror_parent' does not exist 2026-02-22T23:20:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ CEPH_ARGS= 2026-02-22T23:20:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T23:20:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:pool 'mirror_parent' does not exist 2026-02-22T23:20:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T23:20:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirrors cluster1 2026-02-22T23:20:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1 2026-02-22T23:20:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:20:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:20:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:20:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T23:20:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:0 2026-02-22T23:20:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:20:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:20:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:20:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:0 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:0 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:20:11.130 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:20:11.141 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:20:11.142 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:20:11.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=87091 2026-02-22T23:20:11.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 87091 ']' 2026-02-22T23:20:11.143 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 87091 2026-02-22T23:20:11.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:20:11.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:20:11.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.144 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=87091 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:20:11.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.160 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=87091 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T23:20:11.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:20:11.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:20:11.174 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:20:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:20:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:20:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:20:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:20:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:20:11.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:20:11.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:20:11.176 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:20:11.185 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:20:11.186 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:20:11.187 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T23:20:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:0 2026-02-22T23:20:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:20:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T23:20:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:20:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:20:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:20:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:20:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:20:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:20:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:20:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T23:20:11.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:20:11.200 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T23:20:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:0 2026-02-22T23:20:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:0 2026-02-22T23:20:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T23:20:11.202 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:0 2026-02-22T23:20:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:20:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:20:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T23:20:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:20:11.203 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:20:11.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:20:11.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:20:11.204 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T23:20:11.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:20:11.214 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T23:20:11.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:20:11.217 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:1 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:1 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:1 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:20:11.218 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:20:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:20:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:20:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T23:20:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:20:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:20:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:20:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:20:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:20:11.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:20:11.228 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:20:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=87119 2026-02-22T23:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 87119 ']' 2026-02-22T23:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 87119 2026-02-22T23:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:20:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:20:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=87119 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.244 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:20:11.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.245 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=87119 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.260 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:20:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:20:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:20:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:20:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:20:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:20:11.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:20:11.270 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:20:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T23:20:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:1 2026-02-22T23:20:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:20:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T23:20:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:20:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:20:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:20:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:20:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:20:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:20:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:20:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T23:20:11.284 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:20:11.285 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T23:20:11.286 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:1 2026-02-22T23:20:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:1 2026-02-22T23:20:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T23:20:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:1 2026-02-22T23:20:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:20:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:20:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T23:20:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:20:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:20:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:20:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:20:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T23:20:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:20:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T23:20:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T23:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:2 2026-02-22T23:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:20:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T23:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:2 2026-02-22T23:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:2 2026-02-22T23:20:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T23:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:20:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:20:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:20:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:20:11.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=87195 2026-02-22T23:20:11.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 87195 ']' 2026-02-22T23:20:11.313 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 87195 2026-02-22T23:20:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:20:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:20:11.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.315 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=87195 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:20:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.328 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=87195 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:20:11.343 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:20:11.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:20:11.354 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T23:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:2 2026-02-22T23:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:20:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T23:20:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:20:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:2 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:2 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:2 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T23:20:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:20:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T23:20:11.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:20:11.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T23:20:11.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster1:3 2026-02-22T23:20:11.385 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster1:3 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster1:3 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:20:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T23:20:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster1 2026-02-22T23:20:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster1 2026-02-22T23:20:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:20:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:20:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:20:11.396 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:20:11.397 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:20:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=87256 2026-02-22T23:20:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 87256 ']' 2026-02-22T23:20:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 87256 2026-02-22T23:20:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:20:11.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:20:11.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.399 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=87256 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:20:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=87256 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T23:20:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:20:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:20:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:20:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:20:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:20:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:20:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:20:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:20:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:20:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:20:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:20:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:20:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:20:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T23:20:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster1:3 2026-02-22T23:20:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:20:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T23:20:11.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:20:11.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:20:11.444 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:20:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster1 2026-02-22T23:20:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster1 2026-02-22T23:20:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:20:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:20:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T23:20:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:20:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T23:20:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster1:3 2026-02-22T23:20:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1:3 2026-02-22T23:20:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T23:20:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster1:3 2026-02-22T23:20:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:20:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster1 2026-02-22T23:20:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:20:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T23:20:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster1 2026-02-22T23:20:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster1 2026-02-22T23:20:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:20:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:20:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T23:20:11.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:20:11.468 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T23:20:11.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T23:20:11.469 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirrors cluster2 2026-02-22T23:20:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2 2026-02-22T23:20:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:20:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ seq 0 3 2026-02-22T23:20:11.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:20:11.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster2:0 '' 2026-02-22T23:20:11.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:0 2026-02-22T23:20:11.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:20:11.472 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster2:0 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2:0 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:0 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T23:20:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster2 2026-02-22T23:20:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster2 2026-02-22T23:20:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=0 2026-02-22T23:20:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=0 2026-02-22T23:20:11.474 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T23:20:11.483 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T23:20:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T23:20:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=84087 2026-02-22T23:20:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 84087 ']' 2026-02-22T23:20:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 84087 2026-02-22T23:20:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:20:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:20:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=84087 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.501 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:20:11.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.502 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=84087 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-22T23:20:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:0 2026-02-22T23:20:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:20:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:20:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:20:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:20:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:20:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:20:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:20:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:20:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:20:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:20:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:20:11.528 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:20:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T23:20:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:0 2026-02-22T23:20:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:20:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T23:20:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:20:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:20:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:20:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:20:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:20:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:20:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:20:11.531 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T23:20:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:20:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T23:20:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster2:0 2026-02-22T23:20:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:0 2026-02-22T23:20:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:0 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=0 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=0 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=0 2026-02-22T23:20:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T23:20:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T23:20:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T23:20:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:20:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster2:1 '' 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:1 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster2:1 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2:1 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:1 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:20:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:20:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T23:20:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster2 2026-02-22T23:20:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster2 2026-02-22T23:20:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=1 2026-02-22T23:20:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=1 2026-02-22T23:20:11.558 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T23:20:11.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T23:20:11.567 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T23:20:11.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=84131 2026-02-22T23:20:11.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 84131 ']' 2026-02-22T23:20:11.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 84131 2026-02-22T23:20:11.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:20:11.569 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:20:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=84131 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:20:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=84131 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.599 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-02-22T23:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:1 2026-02-22T23:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.600 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:20:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:20:11.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:20:11.611 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:20:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T23:20:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:1 2026-02-22T23:20:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:20:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T23:20:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:20:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:20:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:20:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:20:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:20:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:20:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:20:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T23:20:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:20:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T23:20:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster2:1 2026-02-22T23:20:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:1 2026-02-22T23:20:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T23:20:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:1 2026-02-22T23:20:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:20:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=1 2026-02-22T23:20:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T23:20:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:20:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=1 2026-02-22T23:20:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=1 2026-02-22T23:20:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T23:20:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T23:20:11.637 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T23:20:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:20:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster2:2 '' 2026-02-22T23:20:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:2 2026-02-22T23:20:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:20:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:20:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster2:2 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2:2 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:2 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster2 2026-02-22T23:20:11.640 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster2 2026-02-22T23:20:11.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=2 2026-02-22T23:20:11.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=2 2026-02-22T23:20:11.641 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T23:20:11.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T23:20:11.650 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T23:20:11.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=84192 2026-02-22T23:20:11.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 84192 ']' 2026-02-22T23:20:11.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 84192 2026-02-22T23:20:11.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:20:11.651 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:20:11.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.652 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=84192 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:20:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.667 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=84192 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-02-22T23:20:11.682 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:2 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:20:11.683 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:20:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:20:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:20:11.684 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T23:20:11.695 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:20:11.695 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:20:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T23:20:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:2 2026-02-22T23:20:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:20:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T23:20:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:20:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:20:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:20:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:20:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:20:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:20:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:20:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T23:20:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:20:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T23:20:11.710 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster2:2 2026-02-22T23:20:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:2 2026-02-22T23:20:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T23:20:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:2 2026-02-22T23:20:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:20:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=2 2026-02-22T23:20:11.711 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T23:20:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:20:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=2 2026-02-22T23:20:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=2 2026-02-22T23:20:11.712 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T23:20:11.722 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T23:20:11.722 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T23:20:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T23:20:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ stop_mirror cluster2:3 '' 2026-02-22T23:20:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local cluster=cluster2:3 2026-02-22T23:20:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local sig= 2026-02-22T23:20:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test -n '' 2026-02-22T23:20:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ local pid 2026-02-22T23:20:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ daemon_pid_file cluster2:3 2026-02-22T23:20:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster=cluster2:3 2026-02-22T23:20:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance 2026-02-22T23:20:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:20:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local val=cluster2:3 2026-02-22T23:20:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local cluster_var_name=cluster 2026-02-22T23:20:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local instance_var_name=instance 2026-02-22T23:20:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:20:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T23:20:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval cluster=cluster2 2026-02-22T23:20:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ cluster=cluster2 2026-02-22T23:20:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ eval instance=3 2026-02-22T23:20:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ instance=3 2026-02-22T23:20:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T23:20:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T23:20:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T23:20:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ pid=84260 2026-02-22T23:20:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -n 84260 ']' 2026-02-22T23:20:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ kill 84260 2026-02-22T23:20:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T23:20:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ sleep 0 2026-02-22T23:20:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=84260 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.752 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ break 2026-02-22T23:20:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ ps auxww 2026-02-22T23:20:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ awk -v pid=84260 '$2 == pid {print; exit 1}' 2026-02-22T23:20:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-02-22T23:20:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:3 2026-02-22T23:20:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster1 2026-02-22T23:20:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:20:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T23:20:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:20:11.779 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:20:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T23:20:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local local_cluster=cluster2:3 2026-02-22T23:20:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2 2026-02-22T23:20:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T23:20:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:20:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=local_cluster 2026-02-22T23:20:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:20:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:20:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval local_cluster=cluster2 2026-02-22T23:20:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ local_cluster=cluster2 2026-02-22T23:20:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:20:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:20:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T23:20:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:20:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T23:20:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ daemon_pid_file cluster2:3 2026-02-22T23:20:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster=cluster2:3 2026-02-22T23:20:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local val=cluster2:3 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local cluster_var_name=cluster 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ local instance_var_name=instance 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ cluster=cluster2 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ instance=3 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval cluster=cluster2 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ cluster=cluster2 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ eval instance=3 2026-02-22T23:20:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ instance=3 2026-02-22T23:20:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T23:20:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T23:20:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T23:20:11.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' -z 1 ']' 2026-02-22T23:20:11.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ test '' = /tmp/tmp.VQMrvUqClS 2026-02-22T23:20:11.807 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ rm -Rf /tmp/tmp.VQMrvUqClS 2026-02-22T23:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T23:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ echo OK 2026-02-22T23:20:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial082.stdout:OK 2026-02-22T23:20:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial082.stderr:+ exit 0 2026-02-22T23:20:11.810 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-22T23:20:11.810 DEBUG:teuthology.orchestra.run.trial082:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-22T23:20:11.879 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror.sh'] on cluster1.client.mirror... 2026-02-22T23:20:11.879 DEBUG:teuthology.orchestra.run.trial082:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-22T23:20:12.222 DEBUG:teuthology.parallel:result is None 2026-02-22T23:20:12.222 DEBUG:teuthology.orchestra.run.trial082:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T23:20:12.247 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T23:20:12.248 DEBUG:teuthology.orchestra.run.trial082:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T23:20:12.304 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T23:20:12.305 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T23:20:12.349 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T23:20:12.350 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T23:20:12.508 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T23:20:12.508 INFO:teuthology.orchestra.run.trial082.stderr:dumped all 2026-02-22T23:20:12.516 INFO:teuthology.orchestra.run.trial082.stdout:{"pg_ready":true,"pg_map":{"version":1229,"stamp":"2026-02-22T23:20:11.996325+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":808256,"kb_used_data":419452,"kb_used_omap":4,"kb_used_meta":384635,"kb_avail":282307264,"statfs":{"total":289910292480,"available":289082638336,"internally_reserved":0,"allocated":429518848,"data_stored":423423876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":393866578},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7106224350,"num_objects":-1889,"num_object_clones":-158,"num_object_copies":-3778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1889,"num_whiteouts":0,"num_read":-96659,"num_read_kb":-15952957,"num_write":-17233,"num_write_kb":-24545464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-1,"num_objects_omap":-23,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.939442"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064498+0000","last_change":"2026-02-22T22:57:01.989798+0000","last_active":"2026-02-22T23:20:07.064498+0000","last_peered":"2026-02-22T23:20:07.064498+0000","last_clean":"2026-02-22T23:20:07.064498+0000","last_became_active":"2026-02-22T22:56:59.987009+0000","last_became_peered":"2026-02-22T22:56:59.987009+0000","last_unstale":"2026-02-22T23:20:07.064498+0000","last_undegraded":"2026-02-22T23:20:07.064498+0000","last_fullsized":"2026-02-22T23:20:07.064498+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:37.582516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064088199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064301+0000","last_change":"2026-02-22T22:57:01.989565+0000","last_active":"2026-02-22T23:20:07.064301+0000","last_peered":"2026-02-22T23:20:07.064301+0000","last_clean":"2026-02-22T23:20:07.064301+0000","last_became_active":"2026-02-22T22:56:59.987115+0000","last_became_peered":"2026-02-22T22:56:59.987115+0000","last_unstale":"2026-02-22T23:20:07.064301+0000","last_undegraded":"2026-02-22T23:20:07.064301+0000","last_fullsized":"2026-02-22T23:20:07.064301+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:03:17.983122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000448434,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1054,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063435+0000","last_change":"2026-02-22T22:57:01.989987+0000","last_active":"2026-02-22T23:20:07.063435+0000","last_peered":"2026-02-22T23:20:07.063435+0000","last_clean":"2026-02-22T23:20:07.063435+0000","last_became_active":"2026-02-22T22:56:59.987092+0000","last_became_peered":"2026-02-22T22:56:59.987092+0000","last_unstale":"2026-02-22T23:20:07.063435+0000","last_undegraded":"2026-02-22T23:20:07.063435+0000","last_fullsized":"2026-02-22T23:20:07.063435+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:29:49.602573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000831597,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1054,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063707+0000","last_change":"2026-02-22T22:57:01.989535+0000","last_active":"2026-02-22T23:20:07.063707+0000","last_peered":"2026-02-22T23:20:07.063707+0000","last_clean":"2026-02-22T23:20:07.063707+0000","last_became_active":"2026-02-22T22:56:59.987136+0000","last_became_peered":"2026-02-22T22:56:59.987136+0000","last_unstale":"2026-02-22T23:20:07.063707+0000","last_undegraded":"2026-02-22T23:20:07.063707+0000","last_fullsized":"2026-02-22T23:20:07.063707+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:55.509626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000443876,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063436+0000","last_change":"2026-02-22T22:57:01.991533+0000","last_active":"2026-02-22T23:20:07.063436+0000","last_peered":"2026-02-22T23:20:07.063436+0000","last_clean":"2026-02-22T23:20:07.063436+0000","last_became_active":"2026-02-22T22:56:59.987223+0000","last_became_peered":"2026-02-22T22:56:59.987223+0000","last_unstale":"2026-02-22T23:20:07.063436+0000","last_undegraded":"2026-02-22T23:20:07.063436+0000","last_fullsized":"2026-02-22T23:20:07.063436+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:21:47.758782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051754199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1056,"reported_epoch":530,"state":"active+clean","last_fresh":"2026-02-22T23:20:09.246459+0000","last_change":"2026-02-22T22:57:02.043669+0000","last_active":"2026-02-22T23:20:09.246459+0000","last_peered":"2026-02-22T23:20:09.246459+0000","last_clean":"2026-02-22T23:20:09.246459+0000","last_became_active":"2026-02-22T22:56:59.987301+0000","last_became_peered":"2026-02-22T22:56:59.987301+0000","last_unstale":"2026-02-22T23:20:09.246459+0000","last_undegraded":"2026-02-22T23:20:09.246459+0000","last_fullsized":"2026-02-22T23:20:09.246459+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:02:11.006528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029387700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1056,"reported_epoch":530,"state":"active+clean","last_fresh":"2026-02-22T23:20:09.243603+0000","last_change":"2026-02-22T22:57:02.043628+0000","last_active":"2026-02-22T23:20:09.243603+0000","last_peered":"2026-02-22T23:20:09.243603+0000","last_clean":"2026-02-22T23:20:09.243603+0000","last_became_active":"2026-02-22T22:56:59.987306+0000","last_became_peered":"2026-02-22T22:56:59.987306+0000","last_unstale":"2026-02-22T23:20:09.243603+0000","last_undegraded":"2026-02-22T23:20:09.243603+0000","last_fullsized":"2026-02-22T23:20:09.243603+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:00:55.873154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026463300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":1055,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064118+0000","last_change":"2026-02-22T22:57:01.989963+0000","last_active":"2026-02-22T23:20:07.064118+0000","last_peered":"2026-02-22T23:20:07.064118+0000","last_clean":"2026-02-22T23:20:07.064118+0000","last_became_active":"2026-02-22T22:56:59.986733+0000","last_became_peered":"2026-02-22T22:56:59.986733+0000","last_unstale":"2026-02-22T23:20:07.064118+0000","last_undegraded":"2026-02-22T23:20:07.064118+0000","last_fullsized":"2026-02-22T23:20:07.064118+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:33:04.386924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080243299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":1145,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063266+0000","last_change":"2026-02-22T22:56:56.981052+0000","last_active":"2026-02-22T23:20:07.063266+0000","last_peered":"2026-02-22T23:20:07.063266+0000","last_clean":"2026-02-22T23:20:07.063266+0000","last_became_active":"2026-02-22T22:56:56.980851+0000","last_became_peered":"2026-02-22T22:56:56.980851+0000","last_unstale":"2026-02-22T23:20:07.063266+0000","last_undegraded":"2026-02-22T23:20:07.063266+0000","last_fullsized":"2026-02-22T23:20:07.063266+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_clean_scrub_stamp":"2026-02-22T22:56:55.976608+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:21:14.696753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705948,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":115828,"kb_used_data":4256,"kb_used_omap":1,"kb_used_meta":111550,"kb_avail":94256012,"statfs":{"total":96636764160,"available":96518156288,"internally_reserved":0,"allocated":4358144,"data_stored":2326134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114227654},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705947,"num_pgs":99,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":116404,"kb_used_data":4828,"kb_used_omap":1,"kb_used_meta":111550,"kb_avail":94255436,"statfs":{"total":96636764160,"available":96517566464,"internally_reserved":0,"allocated":4943872,"data_stored":2912391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114227654},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738651,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":576024,"kb_used_data":410368,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":93795816,"statfs":{"total":96636764160,"available":96046915584,"internally_reserved":0,"allocated":420216832,"data_stored":418185351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165411270},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T23:20:12.518 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T23:20:12.677 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T23:20:12.677 INFO:teuthology.orchestra.run.trial082.stderr:dumped all 2026-02-22T23:20:12.685 INFO:teuthology.orchestra.run.trial082.stdout:{"pg_ready":true,"pg_map":{"version":1229,"stamp":"2026-02-22T23:20:11.996325+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":808256,"kb_used_data":419452,"kb_used_omap":4,"kb_used_meta":384635,"kb_avail":282307264,"statfs":{"total":289910292480,"available":289082638336,"internally_reserved":0,"allocated":429518848,"data_stored":423423876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":393866578},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7106224350,"num_objects":-1889,"num_object_clones":-158,"num_object_copies":-3778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1889,"num_whiteouts":0,"num_read":-96659,"num_read_kb":-15952957,"num_write":-17233,"num_write_kb":-24545464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-1,"num_objects_omap":-23,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.939442"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064498+0000","last_change":"2026-02-22T22:57:01.989798+0000","last_active":"2026-02-22T23:20:07.064498+0000","last_peered":"2026-02-22T23:20:07.064498+0000","last_clean":"2026-02-22T23:20:07.064498+0000","last_became_active":"2026-02-22T22:56:59.987009+0000","last_became_peered":"2026-02-22T22:56:59.987009+0000","last_unstale":"2026-02-22T23:20:07.064498+0000","last_undegraded":"2026-02-22T23:20:07.064498+0000","last_fullsized":"2026-02-22T23:20:07.064498+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:37.582516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064088199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064301+0000","last_change":"2026-02-22T22:57:01.989565+0000","last_active":"2026-02-22T23:20:07.064301+0000","last_peered":"2026-02-22T23:20:07.064301+0000","last_clean":"2026-02-22T23:20:07.064301+0000","last_became_active":"2026-02-22T22:56:59.987115+0000","last_became_peered":"2026-02-22T22:56:59.987115+0000","last_unstale":"2026-02-22T23:20:07.064301+0000","last_undegraded":"2026-02-22T23:20:07.064301+0000","last_fullsized":"2026-02-22T23:20:07.064301+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:03:17.983122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000448434,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1054,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063435+0000","last_change":"2026-02-22T22:57:01.989987+0000","last_active":"2026-02-22T23:20:07.063435+0000","last_peered":"2026-02-22T23:20:07.063435+0000","last_clean":"2026-02-22T23:20:07.063435+0000","last_became_active":"2026-02-22T22:56:59.987092+0000","last_became_peered":"2026-02-22T22:56:59.987092+0000","last_unstale":"2026-02-22T23:20:07.063435+0000","last_undegraded":"2026-02-22T23:20:07.063435+0000","last_fullsized":"2026-02-22T23:20:07.063435+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:29:49.602573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000831597,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1054,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063707+0000","last_change":"2026-02-22T22:57:01.989535+0000","last_active":"2026-02-22T23:20:07.063707+0000","last_peered":"2026-02-22T23:20:07.063707+0000","last_clean":"2026-02-22T23:20:07.063707+0000","last_became_active":"2026-02-22T22:56:59.987136+0000","last_became_peered":"2026-02-22T22:56:59.987136+0000","last_unstale":"2026-02-22T23:20:07.063707+0000","last_undegraded":"2026-02-22T23:20:07.063707+0000","last_fullsized":"2026-02-22T23:20:07.063707+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:55.509626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000443876,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063436+0000","last_change":"2026-02-22T22:57:01.991533+0000","last_active":"2026-02-22T23:20:07.063436+0000","last_peered":"2026-02-22T23:20:07.063436+0000","last_clean":"2026-02-22T23:20:07.063436+0000","last_became_active":"2026-02-22T22:56:59.987223+0000","last_became_peered":"2026-02-22T22:56:59.987223+0000","last_unstale":"2026-02-22T23:20:07.063436+0000","last_undegraded":"2026-02-22T23:20:07.063436+0000","last_fullsized":"2026-02-22T23:20:07.063436+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:21:47.758782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051754199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1056,"reported_epoch":530,"state":"active+clean","last_fresh":"2026-02-22T23:20:09.246459+0000","last_change":"2026-02-22T22:57:02.043669+0000","last_active":"2026-02-22T23:20:09.246459+0000","last_peered":"2026-02-22T23:20:09.246459+0000","last_clean":"2026-02-22T23:20:09.246459+0000","last_became_active":"2026-02-22T22:56:59.987301+0000","last_became_peered":"2026-02-22T22:56:59.987301+0000","last_unstale":"2026-02-22T23:20:09.246459+0000","last_undegraded":"2026-02-22T23:20:09.246459+0000","last_fullsized":"2026-02-22T23:20:09.246459+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:02:11.006528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029387700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1056,"reported_epoch":530,"state":"active+clean","last_fresh":"2026-02-22T23:20:09.243603+0000","last_change":"2026-02-22T22:57:02.043628+0000","last_active":"2026-02-22T23:20:09.243603+0000","last_peered":"2026-02-22T23:20:09.243603+0000","last_clean":"2026-02-22T23:20:09.243603+0000","last_became_active":"2026-02-22T22:56:59.987306+0000","last_became_peered":"2026-02-22T22:56:59.987306+0000","last_unstale":"2026-02-22T23:20:09.243603+0000","last_undegraded":"2026-02-22T23:20:09.243603+0000","last_fullsized":"2026-02-22T23:20:09.243603+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:00:55.873154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026463300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":1055,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064118+0000","last_change":"2026-02-22T22:57:01.989963+0000","last_active":"2026-02-22T23:20:07.064118+0000","last_peered":"2026-02-22T23:20:07.064118+0000","last_clean":"2026-02-22T23:20:07.064118+0000","last_became_active":"2026-02-22T22:56:59.986733+0000","last_became_peered":"2026-02-22T22:56:59.986733+0000","last_unstale":"2026-02-22T23:20:07.064118+0000","last_undegraded":"2026-02-22T23:20:07.064118+0000","last_fullsized":"2026-02-22T23:20:07.064118+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:33:04.386924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080243299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":1145,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063266+0000","last_change":"2026-02-22T22:56:56.981052+0000","last_active":"2026-02-22T23:20:07.063266+0000","last_peered":"2026-02-22T23:20:07.063266+0000","last_clean":"2026-02-22T23:20:07.063266+0000","last_became_active":"2026-02-22T22:56:56.980851+0000","last_became_peered":"2026-02-22T22:56:56.980851+0000","last_unstale":"2026-02-22T23:20:07.063266+0000","last_undegraded":"2026-02-22T23:20:07.063266+0000","last_fullsized":"2026-02-22T23:20:07.063266+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_clean_scrub_stamp":"2026-02-22T22:56:55.976608+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:21:14.696753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705948,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":115828,"kb_used_data":4256,"kb_used_omap":1,"kb_used_meta":111550,"kb_avail":94256012,"statfs":{"total":96636764160,"available":96518156288,"internally_reserved":0,"allocated":4358144,"data_stored":2326134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114227654},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705947,"num_pgs":99,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":116404,"kb_used_data":4828,"kb_used_omap":1,"kb_used_meta":111550,"kb_avail":94255436,"statfs":{"total":96636764160,"available":96517566464,"internally_reserved":0,"allocated":4943872,"data_stored":2912391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114227654},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738651,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":576024,"kb_used_data":410368,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":93795816,"statfs":{"total":96636764160,"available":96046915584,"internally_reserved":0,"allocated":420216832,"data_stored":418185351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165411270},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T23:20:12.687 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T23:20:12.687 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T23:20:12.847 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T23:20:12.848 INFO:teuthology.orchestra.run.trial082.stderr:dumped all 2026-02-22T23:20:12.857 INFO:teuthology.orchestra.run.trial082.stdout:{"pg_ready":true,"pg_map":{"version":1229,"stamp":"2026-02-22T23:20:11.996325+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":808256,"kb_used_data":419452,"kb_used_omap":4,"kb_used_meta":384635,"kb_avail":282307264,"statfs":{"total":289910292480,"available":289082638336,"internally_reserved":0,"allocated":429518848,"data_stored":423423876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":393866578},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7106224350,"num_objects":-1889,"num_object_clones":-158,"num_object_copies":-3778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1889,"num_whiteouts":0,"num_read":-96659,"num_read_kb":-15952957,"num_write":-17233,"num_write_kb":-24545464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-1,"num_objects_omap":-23,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.939442"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064498+0000","last_change":"2026-02-22T22:57:01.989798+0000","last_active":"2026-02-22T23:20:07.064498+0000","last_peered":"2026-02-22T23:20:07.064498+0000","last_clean":"2026-02-22T23:20:07.064498+0000","last_became_active":"2026-02-22T22:56:59.987009+0000","last_became_peered":"2026-02-22T22:56:59.987009+0000","last_unstale":"2026-02-22T23:20:07.064498+0000","last_undegraded":"2026-02-22T23:20:07.064498+0000","last_fullsized":"2026-02-22T23:20:07.064498+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:37.582516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064088199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064301+0000","last_change":"2026-02-22T22:57:01.989565+0000","last_active":"2026-02-22T23:20:07.064301+0000","last_peered":"2026-02-22T23:20:07.064301+0000","last_clean":"2026-02-22T23:20:07.064301+0000","last_became_active":"2026-02-22T22:56:59.987115+0000","last_became_peered":"2026-02-22T22:56:59.987115+0000","last_unstale":"2026-02-22T23:20:07.064301+0000","last_undegraded":"2026-02-22T23:20:07.064301+0000","last_fullsized":"2026-02-22T23:20:07.064301+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:03:17.983122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000448434,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1054,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063435+0000","last_change":"2026-02-22T22:57:01.989987+0000","last_active":"2026-02-22T23:20:07.063435+0000","last_peered":"2026-02-22T23:20:07.063435+0000","last_clean":"2026-02-22T23:20:07.063435+0000","last_became_active":"2026-02-22T22:56:59.987092+0000","last_became_peered":"2026-02-22T22:56:59.987092+0000","last_unstale":"2026-02-22T23:20:07.063435+0000","last_undegraded":"2026-02-22T23:20:07.063435+0000","last_fullsized":"2026-02-22T23:20:07.063435+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:29:49.602573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000831597,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1054,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063707+0000","last_change":"2026-02-22T22:57:01.989535+0000","last_active":"2026-02-22T23:20:07.063707+0000","last_peered":"2026-02-22T23:20:07.063707+0000","last_clean":"2026-02-22T23:20:07.063707+0000","last_became_active":"2026-02-22T22:56:59.987136+0000","last_became_peered":"2026-02-22T22:56:59.987136+0000","last_unstale":"2026-02-22T23:20:07.063707+0000","last_undegraded":"2026-02-22T23:20:07.063707+0000","last_fullsized":"2026-02-22T23:20:07.063707+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:55.509626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000443876,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063436+0000","last_change":"2026-02-22T22:57:01.991533+0000","last_active":"2026-02-22T23:20:07.063436+0000","last_peered":"2026-02-22T23:20:07.063436+0000","last_clean":"2026-02-22T23:20:07.063436+0000","last_became_active":"2026-02-22T22:56:59.987223+0000","last_became_peered":"2026-02-22T22:56:59.987223+0000","last_unstale":"2026-02-22T23:20:07.063436+0000","last_undegraded":"2026-02-22T23:20:07.063436+0000","last_fullsized":"2026-02-22T23:20:07.063436+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:21:47.758782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051754199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1056,"reported_epoch":530,"state":"active+clean","last_fresh":"2026-02-22T23:20:09.246459+0000","last_change":"2026-02-22T22:57:02.043669+0000","last_active":"2026-02-22T23:20:09.246459+0000","last_peered":"2026-02-22T23:20:09.246459+0000","last_clean":"2026-02-22T23:20:09.246459+0000","last_became_active":"2026-02-22T22:56:59.987301+0000","last_became_peered":"2026-02-22T22:56:59.987301+0000","last_unstale":"2026-02-22T23:20:09.246459+0000","last_undegraded":"2026-02-22T23:20:09.246459+0000","last_fullsized":"2026-02-22T23:20:09.246459+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:02:11.006528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029387700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1056,"reported_epoch":530,"state":"active+clean","last_fresh":"2026-02-22T23:20:09.243603+0000","last_change":"2026-02-22T22:57:02.043628+0000","last_active":"2026-02-22T23:20:09.243603+0000","last_peered":"2026-02-22T23:20:09.243603+0000","last_clean":"2026-02-22T23:20:09.243603+0000","last_became_active":"2026-02-22T22:56:59.987306+0000","last_became_peered":"2026-02-22T22:56:59.987306+0000","last_unstale":"2026-02-22T23:20:09.243603+0000","last_undegraded":"2026-02-22T23:20:09.243603+0000","last_fullsized":"2026-02-22T23:20:09.243603+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:00:55.873154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026463300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":1055,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064118+0000","last_change":"2026-02-22T22:57:01.989963+0000","last_active":"2026-02-22T23:20:07.064118+0000","last_peered":"2026-02-22T23:20:07.064118+0000","last_clean":"2026-02-22T23:20:07.064118+0000","last_became_active":"2026-02-22T22:56:59.986733+0000","last_became_peered":"2026-02-22T22:56:59.986733+0000","last_unstale":"2026-02-22T23:20:07.064118+0000","last_undegraded":"2026-02-22T23:20:07.064118+0000","last_fullsized":"2026-02-22T23:20:07.064118+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:33:04.386924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080243299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":1145,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063266+0000","last_change":"2026-02-22T22:56:56.981052+0000","last_active":"2026-02-22T23:20:07.063266+0000","last_peered":"2026-02-22T23:20:07.063266+0000","last_clean":"2026-02-22T23:20:07.063266+0000","last_became_active":"2026-02-22T22:56:56.980851+0000","last_became_peered":"2026-02-22T22:56:56.980851+0000","last_unstale":"2026-02-22T23:20:07.063266+0000","last_undegraded":"2026-02-22T23:20:07.063266+0000","last_fullsized":"2026-02-22T23:20:07.063266+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_clean_scrub_stamp":"2026-02-22T22:56:55.976608+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:21:14.696753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705948,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":115828,"kb_used_data":4256,"kb_used_omap":1,"kb_used_meta":111550,"kb_avail":94256012,"statfs":{"total":96636764160,"available":96518156288,"internally_reserved":0,"allocated":4358144,"data_stored":2326134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114227654},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705947,"num_pgs":99,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":116404,"kb_used_data":4828,"kb_used_omap":1,"kb_used_meta":111550,"kb_avail":94255436,"statfs":{"total":96636764160,"available":96517566464,"internally_reserved":0,"allocated":4943872,"data_stored":2912391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114227654},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738651,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":576024,"kb_used_data":410368,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":93795816,"statfs":{"total":96636764160,"available":96046915584,"internally_reserved":0,"allocated":420216832,"data_stored":418185351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165411270},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T23:20:12.858 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T23:20:13.019 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T23:20:13.019 INFO:teuthology.orchestra.run.trial082.stdout:{"epoch":531,"fsid":"159baabb-b8a7-4828-b518-eb266dc5b284","created":"2026-02-22T22:56:43.930984+0000","modified":"2026-02-22T23:20:11.065440+0000","last_up_change":"2026-02-22T22:56:53.975056+0000","last_in_change":"2026-02-22T22:56:44.500294+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:56:55.807109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:56:58.122668+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b46a050a-f95d-4c3b-aa52-8034685b24cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6800","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6801","nonce":4205957132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6802","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6803","nonce":4205957132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6806","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6807","nonce":4205957132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6804","nonce":4205957132},{"type":"v1","addr":"10.20.193.82:6805","nonce":4205957132}]},"public_addr":"10.20.193.82:6801/4205957132","cluster_addr":"10.20.193.82:6803/4205957132","heartbeat_back_addr":"10.20.193.82:6807/4205957132","heartbeat_front_addr":"10.20.193.82:6805/4205957132","state":["exists","up"]},{"osd":1,"uuid":"7485287f-05fb-4da0-b9dc-4b04e779999d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6808","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6810","nonce":920487848}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6813","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6815","nonce":920487848}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6821","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6823","nonce":920487848}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6817","nonce":920487848},{"type":"v1","addr":"10.20.193.82:6819","nonce":920487848}]},"public_addr":"10.20.193.82:6810/920487848","cluster_addr":"10.20.193.82:6815/920487848","heartbeat_back_addr":"10.20.193.82:6823/920487848","heartbeat_front_addr":"10.20.193.82:6819/920487848","state":["exists","up"]},{"osd":2,"uuid":"953c1811-ccbc-4b30-9679-3c746f040954","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6809","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6811","nonce":1962775415}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6812","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6814","nonce":1962775415}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6820","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6822","nonce":1962775415}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.82:6816","nonce":1962775415},{"type":"v1","addr":"10.20.193.82:6818","nonce":1962775415}]},"public_addr":"10.20.193.82:6811/1962775415","cluster_addr":"10.20.193.82:6814/1962775415","heartbeat_back_addr":"10.20.193.82:6822/1962775415","heartbeat_front_addr":"10.20.193.82:6818/1962775415","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.868297+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.957575+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:51.971102+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":198,"length":3}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T23:20:14.030 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-22T23:20:14.030 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T23:20:14.113 INFO:teuthology.orchestra.run.trial082.stdout:{ 2026-02-22T23:20:14.113 INFO:teuthology.orchestra.run.trial082.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T23:20:14.113 INFO:teuthology.orchestra.run.trial082.stdout:} 2026-02-22T23:20:14.118 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-22T23:20:14.277 INFO:teuthology.orchestra.run.trial082.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T23:20:14.285 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-22T23:20:14.285 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T23:20:14.367 INFO:teuthology.orchestra.run.trial082.stdout:{ 2026-02-22T23:20:14.367 INFO:teuthology.orchestra.run.trial082.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T23:20:14.367 INFO:teuthology.orchestra.run.trial082.stdout:} 2026-02-22T23:20:14.372 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-22T23:20:14.533 INFO:teuthology.orchestra.run.trial082.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T23:20:14.542 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-22T23:20:14.542 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T23:20:14.623 INFO:teuthology.orchestra.run.trial082.stdout:{ 2026-02-22T23:20:14.623 INFO:teuthology.orchestra.run.trial082.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T23:20:14.624 INFO:teuthology.orchestra.run.trial082.stdout:} 2026-02-22T23:20:14.629 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-22T23:20:14.787 INFO:teuthology.orchestra.run.trial082.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T23:20:14.795 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T23:20:14.952 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T23:20:14.953 INFO:teuthology.orchestra.run.trial082.stderr:dumped all 2026-02-22T23:20:14.960 INFO:teuthology.orchestra.run.trial082.stdout:{"pg_ready":true,"pg_map":{"version":1230,"stamp":"2026-02-22T23:20:13.996496+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":186,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":398636,"kb_used_data":13932,"kb_used_omap":4,"kb_used_meta":384635,"kb_avail":282716884,"statfs":{"total":289910292480,"available":289502089216,"internally_reserved":0,"allocated":14266368,"data_stored":8158683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":393866578},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7106224350,"num_objects":-1889,"num_object_clones":-158,"num_object_copies":-3778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1889,"num_whiteouts":0,"num_read":-96659,"num_read_kb":-15952957,"num_write":-17233,"num_write_kb":-24545464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-1,"num_objects_omap":-23,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.936523"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064498+0000","last_change":"2026-02-22T22:57:01.989798+0000","last_active":"2026-02-22T23:20:07.064498+0000","last_peered":"2026-02-22T23:20:07.064498+0000","last_clean":"2026-02-22T23:20:07.064498+0000","last_became_active":"2026-02-22T22:56:59.987009+0000","last_became_peered":"2026-02-22T22:56:59.987009+0000","last_unstale":"2026-02-22T23:20:07.064498+0000","last_undegraded":"2026-02-22T23:20:07.064498+0000","last_fullsized":"2026-02-22T23:20:07.064498+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:37.582516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064088199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1053,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064301+0000","last_change":"2026-02-22T22:57:01.989565+0000","last_active":"2026-02-22T23:20:07.064301+0000","last_peered":"2026-02-22T23:20:07.064301+0000","last_clean":"2026-02-22T23:20:07.064301+0000","last_became_active":"2026-02-22T22:56:59.987115+0000","last_became_peered":"2026-02-22T22:56:59.987115+0000","last_unstale":"2026-02-22T23:20:07.064301+0000","last_undegraded":"2026-02-22T23:20:07.064301+0000","last_fullsized":"2026-02-22T23:20:07.064301+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:03:17.983122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000448434,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1054,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063435+0000","last_change":"2026-02-22T22:57:01.989987+0000","last_active":"2026-02-22T23:20:07.063435+0000","last_peered":"2026-02-22T23:20:07.063435+0000","last_clean":"2026-02-22T23:20:07.063435+0000","last_became_active":"2026-02-22T22:56:59.987092+0000","last_became_peered":"2026-02-22T22:56:59.987092+0000","last_unstale":"2026-02-22T23:20:07.063435+0000","last_undegraded":"2026-02-22T23:20:07.063435+0000","last_fullsized":"2026-02-22T23:20:07.063435+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:29:49.602573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000831597,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1054,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063707+0000","last_change":"2026-02-22T22:57:01.989535+0000","last_active":"2026-02-22T23:20:07.063707+0000","last_peered":"2026-02-22T23:20:07.063707+0000","last_clean":"2026-02-22T23:20:07.063707+0000","last_became_active":"2026-02-22T22:56:59.987136+0000","last_became_peered":"2026-02-22T22:56:59.987136+0000","last_unstale":"2026-02-22T23:20:07.063707+0000","last_undegraded":"2026-02-22T23:20:07.063707+0000","last_fullsized":"2026-02-22T23:20:07.063707+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:03:55.509626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000443876,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":1057,"reported_epoch":531,"state":"active+clean","last_fresh":"2026-02-22T23:20:12.061801+0000","last_change":"2026-02-22T22:57:01.991533+0000","last_active":"2026-02-22T23:20:12.061801+0000","last_peered":"2026-02-22T23:20:12.061801+0000","last_clean":"2026-02-22T23:20:12.061801+0000","last_became_active":"2026-02-22T22:56:59.987223+0000","last_became_peered":"2026-02-22T22:56:59.987223+0000","last_unstale":"2026-02-22T23:20:12.061801+0000","last_undegraded":"2026-02-22T23:20:12.061801+0000","last_fullsized":"2026-02-22T23:20:12.061801+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:21:47.758782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051754199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1056,"reported_epoch":530,"state":"active+clean","last_fresh":"2026-02-22T23:20:09.246459+0000","last_change":"2026-02-22T22:57:02.043669+0000","last_active":"2026-02-22T23:20:09.246459+0000","last_peered":"2026-02-22T23:20:09.246459+0000","last_clean":"2026-02-22T23:20:09.246459+0000","last_became_active":"2026-02-22T22:56:59.987301+0000","last_became_peered":"2026-02-22T22:56:59.987301+0000","last_unstale":"2026-02-22T23:20:09.246459+0000","last_undegraded":"2026-02-22T23:20:09.246459+0000","last_fullsized":"2026-02-22T23:20:09.246459+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:02:11.006528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029387700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1056,"reported_epoch":530,"state":"active+clean","last_fresh":"2026-02-22T23:20:09.243603+0000","last_change":"2026-02-22T22:57:02.043628+0000","last_active":"2026-02-22T23:20:09.243603+0000","last_peered":"2026-02-22T23:20:09.243603+0000","last_clean":"2026-02-22T23:20:09.243603+0000","last_became_active":"2026-02-22T22:56:59.987306+0000","last_became_peered":"2026-02-22T22:56:59.987306+0000","last_unstale":"2026-02-22T23:20:09.243603+0000","last_undegraded":"2026-02-22T23:20:09.243603+0000","last_fullsized":"2026-02-22T23:20:09.243603+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:00:55.873154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026463300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":1055,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.064118+0000","last_change":"2026-02-22T22:57:01.989963+0000","last_active":"2026-02-22T23:20:07.064118+0000","last_peered":"2026-02-22T23:20:07.064118+0000","last_clean":"2026-02-22T23:20:07.064118+0000","last_became_active":"2026-02-22T22:56:59.986733+0000","last_became_peered":"2026-02-22T22:56:59.986733+0000","last_unstale":"2026-02-22T23:20:07.064118+0000","last_undegraded":"2026-02-22T23:20:07.064118+0000","last_fullsized":"2026-02-22T23:20:07.064118+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:58.981892+0000","last_clean_scrub_stamp":"2026-02-22T22:56:58.981892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:33:04.386924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080243299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":1145,"reported_epoch":529,"state":"active+clean","last_fresh":"2026-02-22T23:20:07.063266+0000","last_change":"2026-02-22T22:56:56.981052+0000","last_active":"2026-02-22T23:20:07.063266+0000","last_peered":"2026-02-22T23:20:07.063266+0000","last_clean":"2026-02-22T23:20:07.063266+0000","last_became_active":"2026-02-22T22:56:56.980851+0000","last_became_peered":"2026-02-22T22:56:56.980851+0000","last_unstale":"2026-02-22T23:20:07.063266+0000","last_undegraded":"2026-02-22T23:20:07.063266+0000","last_fullsized":"2026-02-22T23:20:07.063266+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:55.976608+0000","last_clean_scrub_stamp":"2026-02-22T22:56:55.976608+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:21:14.696753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705948,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":115828,"kb_used_data":4256,"kb_used_omap":1,"kb_used_meta":111550,"kb_avail":94256012,"statfs":{"total":96636764160,"available":96518156288,"internally_reserved":0,"allocated":4358144,"data_stored":2326134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114227654},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705947,"num_pgs":99,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":116404,"kb_used_data":4828,"kb_used_omap":1,"kb_used_meta":111550,"kb_avail":94255436,"statfs":{"total":96636764160,"available":96517566464,"internally_reserved":0,"allocated":4943872,"data_stored":2912391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114227654},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738652,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":166404,"kb_used_data":4848,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":94205436,"statfs":{"total":96636764160,"available":96466366464,"internally_reserved":0,"allocated":4964352,"data_stored":2920158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165411270},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T23:20:14.965 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T22:56:58.981892+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=58, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=13, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:14.966 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T22:56:58.981892+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=58, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=13, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:14.966 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T22:56:58.981892+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=58, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=13, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:14.966 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T22:56:58.981892+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=58, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=13, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:14.966 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T22:56:58.981892+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=58, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=13, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:14.966 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T22:56:58.981892+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=58, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=13, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:14.966 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T22:56:58.981892+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=58, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=13, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:14.967 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T22:56:58.981892+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=58, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=13, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:14.967 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T22:56:55.976608+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=55, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=13, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:14.967 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T23:20:34.967 DEBUG:teuthology.orchestra.run.trial082:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T23:20:35.128 INFO:teuthology.orchestra.run.trial082.stdout: 2026-02-22T23:20:35.129 INFO:teuthology.orchestra.run.trial082.stderr:dumped all 2026-02-22T23:20:35.138 INFO:teuthology.orchestra.run.trial082.stdout:{"pg_ready":true,"pg_map":{"version":1240,"stamp":"2026-02-22T23:20:33.997988+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":405112,"kb_used_data":13960,"kb_used_omap":4,"kb_used_meta":391099,"kb_avail":282710408,"statfs":{"total":289910292480,"available":289495457792,"internally_reserved":0,"allocated":14295040,"data_stored":8170087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":400485714},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000930"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1066,"reported_epoch":531,"state":"active+clean","last_fresh":"2026-02-22T23:20:16.461903+0000","last_change":"2026-02-22T23:20:16.461923+0000","last_active":"2026-02-22T23:20:16.461903+0000","last_peered":"2026-02-22T23:20:16.461903+0000","last_clean":"2026-02-22T23:20:16.461903+0000","last_became_active":"2026-02-22T22:56:59.987009+0000","last_became_peered":"2026-02-22T22:56:59.987009+0000","last_unstale":"2026-02-22T23:20:16.461903+0000","last_undegraded":"2026-02-22T23:20:16.461903+0000","last_fullsized":"2026-02-22T23:20:16.461903+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:16.461837+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:16.461837+0000","last_clean_scrub_stamp":"2026-02-22T23:20:16.461837+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:37:53.159034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064088199999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1066,"reported_epoch":531,"state":"active+clean","last_fresh":"2026-02-22T23:20:20.467304+0000","last_change":"2026-02-22T23:20:20.467325+0000","last_active":"2026-02-22T23:20:20.467304+0000","last_peered":"2026-02-22T23:20:20.467304+0000","last_clean":"2026-02-22T23:20:20.467304+0000","last_became_active":"2026-02-22T22:56:59.987115+0000","last_became_peered":"2026-02-22T22:56:59.987115+0000","last_unstale":"2026-02-22T23:20:20.467304+0000","last_undegraded":"2026-02-22T23:20:20.467304+0000","last_fullsized":"2026-02-22T23:20:20.467304+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:20.467236+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:20.467236+0000","last_clean_scrub_stamp":"2026-02-22T23:20:20.467236+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:21:39.053454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000448434,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1067,"reported_epoch":531,"state":"active+clean","last_fresh":"2026-02-22T23:20:17.480455+0000","last_change":"2026-02-22T23:20:17.480465+0000","last_active":"2026-02-22T23:20:17.480455+0000","last_peered":"2026-02-22T23:20:17.480455+0000","last_clean":"2026-02-22T23:20:17.480455+0000","last_became_active":"2026-02-22T22:56:59.987092+0000","last_became_peered":"2026-02-22T22:56:59.987092+0000","last_unstale":"2026-02-22T23:20:17.480455+0000","last_undegraded":"2026-02-22T23:20:17.480455+0000","last_fullsized":"2026-02-22T23:20:17.480455+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:17.480408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:17.480408+0000","last_clean_scrub_stamp":"2026-02-22T23:20:17.480408+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:17:17.967741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000831597,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1067,"reported_epoch":531,"state":"active+clean","last_fresh":"2026-02-22T23:20:18.446294+0000","last_change":"2026-02-22T23:20:18.446305+0000","last_active":"2026-02-22T23:20:18.446294+0000","last_peered":"2026-02-22T23:20:18.446294+0000","last_clean":"2026-02-22T23:20:18.446294+0000","last_became_active":"2026-02-22T22:56:59.987136+0000","last_became_peered":"2026-02-22T22:56:59.987136+0000","last_unstale":"2026-02-22T23:20:18.446294+0000","last_undegraded":"2026-02-22T23:20:18.446294+0000","last_fullsized":"2026-02-22T23:20:18.446294+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:18.446253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:18.446253+0000","last_clean_scrub_stamp":"2026-02-22T23:20:18.446253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:14:35.772455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000443876,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"531'3","reported_seq":1067,"reported_epoch":531,"state":"active+clean","last_fresh":"2026-02-22T23:20:14.531711+0000","last_change":"2026-02-22T23:20:14.531726+0000","last_active":"2026-02-22T23:20:14.531711+0000","last_peered":"2026-02-22T23:20:14.531711+0000","last_clean":"2026-02-22T23:20:14.531711+0000","last_became_active":"2026-02-22T22:56:59.987223+0000","last_became_peered":"2026-02-22T22:56:59.987223+0000","last_unstale":"2026-02-22T23:20:14.531711+0000","last_undegraded":"2026-02-22T23:20:14.531711+0000","last_fullsized":"2026-02-22T23:20:14.531711+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"531'3","last_scrub_stamp":"2026-02-22T23:20:14.531669+0000","last_deep_scrub":"531'3","last_deep_scrub_stamp":"2026-02-22T23:20:14.531669+0000","last_clean_scrub_stamp":"2026-02-22T23:20:14.531669+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:12:37.990656+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00051754199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1067,"reported_epoch":531,"state":"active+clean","last_fresh":"2026-02-22T23:20:15.445852+0000","last_change":"2026-02-22T23:20:15.445867+0000","last_active":"2026-02-22T23:20:15.445852+0000","last_peered":"2026-02-22T23:20:15.445852+0000","last_clean":"2026-02-22T23:20:15.445852+0000","last_became_active":"2026-02-22T22:56:59.987301+0000","last_became_peered":"2026-02-22T22:56:59.987301+0000","last_unstale":"2026-02-22T23:20:15.445852+0000","last_undegraded":"2026-02-22T23:20:15.445852+0000","last_fullsized":"2026-02-22T23:20:15.445852+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:15.445772+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:15.445772+0000","last_clean_scrub_stamp":"2026-02-22T23:20:15.445772+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:12:57.107123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029387700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1067,"reported_epoch":531,"state":"active+clean","last_fresh":"2026-02-22T23:20:16.409421+0000","last_change":"2026-02-22T23:20:16.409440+0000","last_active":"2026-02-22T23:20:16.409421+0000","last_peered":"2026-02-22T23:20:16.409421+0000","last_clean":"2026-02-22T23:20:16.409421+0000","last_became_active":"2026-02-22T22:56:59.987306+0000","last_became_peered":"2026-02-22T22:56:59.987306+0000","last_unstale":"2026-02-22T23:20:16.409421+0000","last_undegraded":"2026-02-22T23:20:16.409421+0000","last_fullsized":"2026-02-22T23:20:16.409421+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:16.409323+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:16.409323+0000","last_clean_scrub_stamp":"2026-02-22T23:20:16.409323+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:00:15.141316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026463300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"531'2","reported_seq":1069,"reported_epoch":531,"state":"active+clean","last_fresh":"2026-02-22T23:20:19.458608+0000","last_change":"2026-02-22T23:20:19.458623+0000","last_active":"2026-02-22T23:20:19.458608+0000","last_peered":"2026-02-22T23:20:19.458608+0000","last_clean":"2026-02-22T23:20:19.458608+0000","last_became_active":"2026-02-22T22:56:59.986733+0000","last_became_peered":"2026-02-22T22:56:59.986733+0000","last_unstale":"2026-02-22T23:20:19.458608+0000","last_undegraded":"2026-02-22T23:20:19.458608+0000","last_fullsized":"2026-02-22T23:20:19.458608+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"531'2","last_scrub_stamp":"2026-02-22T23:20:19.458571+0000","last_deep_scrub":"531'2","last_deep_scrub_stamp":"2026-02-22T23:20:19.458571+0000","last_clean_scrub_stamp":"2026-02-22T23:20:19.458571+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:13:34.367663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080243299999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"531'74","reported_seq":1160,"reported_epoch":531,"state":"active+clean","last_fresh":"2026-02-22T23:20:15.498706+0000","last_change":"2026-02-22T23:20:15.498734+0000","last_active":"2026-02-22T23:20:15.498706+0000","last_peered":"2026-02-22T23:20:15.498706+0000","last_clean":"2026-02-22T23:20:15.498706+0000","last_became_active":"2026-02-22T22:56:56.980851+0000","last_became_peered":"2026-02-22T22:56:56.980851+0000","last_unstale":"2026-02-22T23:20:15.498706+0000","last_undegraded":"2026-02-22T23:20:15.498706+0000","last_fullsized":"2026-02-22T23:20:15.498706+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"531'74","last_scrub_stamp":"2026-02-22T23:20:15.498623+0000","last_deep_scrub":"531'74","last_deep_scrub_stamp":"2026-02-22T23:20:15.498623+0000","last_clean_scrub_stamp":"2026-02-22T23:20:15.498623+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:04:21.645714+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705952,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":119992,"kb_used_data":4264,"kb_used_omap":1,"kb_used_meta":115710,"kb_avail":94251848,"statfs":{"total":96636764160,"available":96513892352,"internally_reserved":0,"allocated":4366336,"data_stored":2329771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":118487494},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705951,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":118720,"kb_used_data":4848,"kb_used_omap":1,"kb_used_meta":113854,"kb_avail":94253120,"statfs":{"total":96636764160,"available":96515194880,"internally_reserved":0,"allocated":4964352,"data_stored":2920158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":116586950},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738656,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":166400,"kb_used_data":4848,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":94205440,"statfs":{"total":96636764160,"available":96466370560,"internally_reserved":0,"allocated":4964352,"data_stored":2920158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165411270},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T23:20:35.140 DEBUG:teuthology.orchestra.run.trial082:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-22T23:20:35.308 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T23:20:35.308 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T23:20:35.308 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-22T23:20:35.308 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T23:20:35.368 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-22T23:20:35.368 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-22T23:20:35.368 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T23:20:35.419 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-22T23:20:35.420 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-22T23:20:35.420 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T23:20:35.462 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-22T23:20:35.463 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T23:20:35.463 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-22T23:20:35.463 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T23:20:35.517 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-22T23:20:35.517 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T23:20:35.518 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-22T23:20:35.518 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T23:20:35.530 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-22T23:20:35.530 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T23:20:35.531 DEBUG:teuthology.orchestra.run.trial082:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T23:20:35.596 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial082.front.sepia.ceph.com 2026-02-22T23:20:35.596 DEBUG:teuthology.orchestra.run.trial082:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-22T23:20:35.685 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial082.front.sepia.ceph.com 2026-02-22T23:20:35.685 DEBUG:teuthology.orchestra.run.trial082:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-22T23:20:35.720 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial082.front.sepia.ceph.com 2026-02-22T23:20:35.720 DEBUG:teuthology.orchestra.run.trial082:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-22T23:20:35.800 INFO:tasks.ceph:Archiving mon data... 2026-02-22T23:20:35.802 DEBUG:teuthology.misc:Transferring archived files from trial082:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284/data/cluster2.mon.a.tgz 2026-02-22T23:20:35.803 DEBUG:teuthology.orchestra.run.trial082:> mktemp 2026-02-22T23:20:35.857 INFO:teuthology.orchestra.run.trial082.stdout:/tmp/tmp.ldhopa4qcs 2026-02-22T23:20:35.858 DEBUG:teuthology.orchestra.run.trial082:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.ldhopa4qcs 2026-02-22T23:20:35.993 DEBUG:teuthology.orchestra.run.trial082:> sudo chmod 0666 /tmp/tmp.ldhopa4qcs 2026-02-22T23:20:36.076 DEBUG:teuthology.orchestra.remote:trial082:/tmp/tmp.ldhopa4qcs is 562KB 2026-02-22T23:20:36.176 DEBUG:teuthology.orchestra.run.trial082:> rm -fr /tmp/tmp.ldhopa4qcs 2026-02-22T23:20:36.193 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T23:20:36.193 DEBUG:teuthology.orchestra.run.trial060:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T23:20:36.195 DEBUG:teuthology.orchestra.run.trial082:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T23:20:36.257 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T23:20:36.277 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T23:20:36.277 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T23:20:36.433 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T23:20:36.433 INFO:teuthology.orchestra.run.trial060.stderr:dumped all 2026-02-22T23:20:36.442 INFO:teuthology.orchestra.run.trial060.stdout:{"pg_ready":true,"pg_map":{"version":1288,"stamp":"2026-02-22T23:20:35.418162+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":315140,"kb_used_data":14920,"kb_used_omap":4,"kb_used_meta":300155,"kb_avail":282800380,"statfs":{"total":289910292480,"available":289587589120,"internally_reserved":0,"allocated":15278080,"data_stored":8581588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":307359060},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000922"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.062791+0000","last_change":"2026-02-22T22:56:20.934556+0000","last_active":"2026-02-22T23:20:10.062791+0000","last_peered":"2026-02-22T23:20:10.062791+0000","last_clean":"2026-02-22T23:20:10.062791+0000","last_became_active":"2026-02-22T22:56:18.932408+0000","last_became_peered":"2026-02-22T22:56:18.932408+0000","last_unstale":"2026-02-22T23:20:10.062791+0000","last_undegraded":"2026-02-22T23:20:10.062791+0000","last_fullsized":"2026-02-22T23:20:10.062791+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:45:37.059532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052600900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.064272+0000","last_change":"2026-02-22T22:56:20.934743+0000","last_active":"2026-02-22T23:20:10.064272+0000","last_peered":"2026-02-22T23:20:10.064272+0000","last_clean":"2026-02-22T23:20:10.064272+0000","last_became_active":"2026-02-22T22:56:18.931993+0000","last_became_peered":"2026-02-22T22:56:18.931993+0000","last_unstale":"2026-02-22T23:20:10.064272+0000","last_undegraded":"2026-02-22T23:20:10.064272+0000","last_fullsized":"2026-02-22T23:20:10.064272+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:55.729675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071762499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.063967+0000","last_change":"2026-02-22T22:56:20.934280+0000","last_active":"2026-02-22T23:20:10.063967+0000","last_peered":"2026-02-22T23:20:10.063967+0000","last_clean":"2026-02-22T23:20:10.063967+0000","last_became_active":"2026-02-22T22:56:18.932505+0000","last_became_peered":"2026-02-22T22:56:18.932505+0000","last_unstale":"2026-02-22T23:20:10.063967+0000","last_undegraded":"2026-02-22T23:20:10.063967+0000","last_fullsized":"2026-02-22T23:20:10.063967+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:50:12.129550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000270864,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.062903+0000","last_change":"2026-02-22T22:56:20.934550+0000","last_active":"2026-02-22T23:20:10.062903+0000","last_peered":"2026-02-22T23:20:10.062903+0000","last_clean":"2026-02-22T23:20:10.062903+0000","last_became_active":"2026-02-22T22:56:18.932003+0000","last_became_peered":"2026-02-22T22:56:18.932003+0000","last_unstale":"2026-02-22T23:20:10.062903+0000","last_undegraded":"2026-02-22T23:20:10.062903+0000","last_fullsized":"2026-02-22T23:20:10.062903+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:26:14.433190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054626700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1116,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.516188+0000","last_change":"2026-02-22T22:56:20.932755+0000","last_active":"2026-02-22T23:20:10.516188+0000","last_peered":"2026-02-22T23:20:10.516188+0000","last_clean":"2026-02-22T23:20:10.516188+0000","last_became_active":"2026-02-22T22:56:18.932278+0000","last_became_peered":"2026-02-22T22:56:18.932278+0000","last_unstale":"2026-02-22T23:20:10.516188+0000","last_undegraded":"2026-02-22T23:20:10.516188+0000","last_fullsized":"2026-02-22T23:20:10.516188+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:54:51.140871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066678099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1113,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.076906+0000","last_change":"2026-02-22T22:56:21.006101+0000","last_active":"2026-02-22T23:20:10.076906+0000","last_peered":"2026-02-22T23:20:10.076906+0000","last_clean":"2026-02-22T23:20:10.076906+0000","last_became_active":"2026-02-22T22:56:18.932279+0000","last_became_peered":"2026-02-22T22:56:18.932279+0000","last_unstale":"2026-02-22T23:20:10.076906+0000","last_undegraded":"2026-02-22T23:20:10.076906+0000","last_fullsized":"2026-02-22T23:20:10.076906+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:12:56.667350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023402500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.075534+0000","last_change":"2026-02-22T22:56:21.006207+0000","last_active":"2026-02-22T23:20:10.075534+0000","last_peered":"2026-02-22T23:20:10.075534+0000","last_clean":"2026-02-22T23:20:10.075534+0000","last_became_active":"2026-02-22T22:56:18.931984+0000","last_became_peered":"2026-02-22T22:56:18.931984+0000","last_unstale":"2026-02-22T23:20:10.075534+0000","last_undegraded":"2026-02-22T23:20:10.075534+0000","last_fullsized":"2026-02-22T23:20:10.075534+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:36.205745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033533400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1115,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.063733+0000","last_change":"2026-02-22T22:56:20.934279+0000","last_active":"2026-02-22T23:20:10.063733+0000","last_peered":"2026-02-22T23:20:10.063733+0000","last_clean":"2026-02-22T23:20:10.063733+0000","last_became_active":"2026-02-22T22:56:18.931925+0000","last_became_peered":"2026-02-22T22:56:18.931925+0000","last_unstale":"2026-02-22T23:20:10.063733+0000","last_undegraded":"2026-02-22T23:20:10.063733+0000","last_fullsized":"2026-02-22T23:20:10.063733+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:28:58.350535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000296552,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1206,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.060915+0000","last_change":"2026-02-22T22:56:14.922932+0000","last_active":"2026-02-22T23:20:10.060915+0000","last_peered":"2026-02-22T23:20:10.060915+0000","last_clean":"2026-02-22T23:20:10.060915+0000","last_became_active":"2026-02-22T22:56:14.921193+0000","last_became_peered":"2026-02-22T22:56:14.921193+0000","last_unstale":"2026-02-22T23:20:10.060915+0000","last_undegraded":"2026-02-22T23:20:10.060915+0000","last_fullsized":"2026-02-22T23:20:10.060915+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_clean_scrub_stamp":"2026-02-22T22:56:13.916808+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:23:02.729111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705959,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89852,"kb_used_data":4584,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94281988,"statfs":{"total":96636764160,"available":96544755712,"internally_reserved":0,"allocated":4694016,"data_stored":2466938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":87292359},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705959,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":108484,"kb_used_data":5168,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":94263356,"statfs":{"total":96636764160,"available":96525676544,"internally_reserved":0,"allocated":5292032,"data_stored":3057325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":105773511},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738664,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":116804,"kb_used_data":5168,"kb_used_omap":1,"kb_used_meta":111614,"kb_avail":94255036,"statfs":{"total":96636764160,"available":96517156864,"internally_reserved":0,"allocated":5292032,"data_stored":3057325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114293190},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T23:20:36.442 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T23:20:36.597 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T23:20:36.597 INFO:teuthology.orchestra.run.trial060.stderr:dumped all 2026-02-22T23:20:36.605 INFO:teuthology.orchestra.run.trial060.stdout:{"pg_ready":true,"pg_map":{"version":1288,"stamp":"2026-02-22T23:20:35.418162+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":315140,"kb_used_data":14920,"kb_used_omap":4,"kb_used_meta":300155,"kb_avail":282800380,"statfs":{"total":289910292480,"available":289587589120,"internally_reserved":0,"allocated":15278080,"data_stored":8581588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":307359060},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000922"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.062791+0000","last_change":"2026-02-22T22:56:20.934556+0000","last_active":"2026-02-22T23:20:10.062791+0000","last_peered":"2026-02-22T23:20:10.062791+0000","last_clean":"2026-02-22T23:20:10.062791+0000","last_became_active":"2026-02-22T22:56:18.932408+0000","last_became_peered":"2026-02-22T22:56:18.932408+0000","last_unstale":"2026-02-22T23:20:10.062791+0000","last_undegraded":"2026-02-22T23:20:10.062791+0000","last_fullsized":"2026-02-22T23:20:10.062791+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:45:37.059532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052600900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.064272+0000","last_change":"2026-02-22T22:56:20.934743+0000","last_active":"2026-02-22T23:20:10.064272+0000","last_peered":"2026-02-22T23:20:10.064272+0000","last_clean":"2026-02-22T23:20:10.064272+0000","last_became_active":"2026-02-22T22:56:18.931993+0000","last_became_peered":"2026-02-22T22:56:18.931993+0000","last_unstale":"2026-02-22T23:20:10.064272+0000","last_undegraded":"2026-02-22T23:20:10.064272+0000","last_fullsized":"2026-02-22T23:20:10.064272+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:55.729675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071762499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.063967+0000","last_change":"2026-02-22T22:56:20.934280+0000","last_active":"2026-02-22T23:20:10.063967+0000","last_peered":"2026-02-22T23:20:10.063967+0000","last_clean":"2026-02-22T23:20:10.063967+0000","last_became_active":"2026-02-22T22:56:18.932505+0000","last_became_peered":"2026-02-22T22:56:18.932505+0000","last_unstale":"2026-02-22T23:20:10.063967+0000","last_undegraded":"2026-02-22T23:20:10.063967+0000","last_fullsized":"2026-02-22T23:20:10.063967+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:50:12.129550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000270864,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.062903+0000","last_change":"2026-02-22T22:56:20.934550+0000","last_active":"2026-02-22T23:20:10.062903+0000","last_peered":"2026-02-22T23:20:10.062903+0000","last_clean":"2026-02-22T23:20:10.062903+0000","last_became_active":"2026-02-22T22:56:18.932003+0000","last_became_peered":"2026-02-22T22:56:18.932003+0000","last_unstale":"2026-02-22T23:20:10.062903+0000","last_undegraded":"2026-02-22T23:20:10.062903+0000","last_fullsized":"2026-02-22T23:20:10.062903+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:26:14.433190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054626700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1116,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.516188+0000","last_change":"2026-02-22T22:56:20.932755+0000","last_active":"2026-02-22T23:20:10.516188+0000","last_peered":"2026-02-22T23:20:10.516188+0000","last_clean":"2026-02-22T23:20:10.516188+0000","last_became_active":"2026-02-22T22:56:18.932278+0000","last_became_peered":"2026-02-22T22:56:18.932278+0000","last_unstale":"2026-02-22T23:20:10.516188+0000","last_undegraded":"2026-02-22T23:20:10.516188+0000","last_fullsized":"2026-02-22T23:20:10.516188+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:54:51.140871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066678099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1113,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.076906+0000","last_change":"2026-02-22T22:56:21.006101+0000","last_active":"2026-02-22T23:20:10.076906+0000","last_peered":"2026-02-22T23:20:10.076906+0000","last_clean":"2026-02-22T23:20:10.076906+0000","last_became_active":"2026-02-22T22:56:18.932279+0000","last_became_peered":"2026-02-22T22:56:18.932279+0000","last_unstale":"2026-02-22T23:20:10.076906+0000","last_undegraded":"2026-02-22T23:20:10.076906+0000","last_fullsized":"2026-02-22T23:20:10.076906+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:12:56.667350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023402500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.075534+0000","last_change":"2026-02-22T22:56:21.006207+0000","last_active":"2026-02-22T23:20:10.075534+0000","last_peered":"2026-02-22T23:20:10.075534+0000","last_clean":"2026-02-22T23:20:10.075534+0000","last_became_active":"2026-02-22T22:56:18.931984+0000","last_became_peered":"2026-02-22T22:56:18.931984+0000","last_unstale":"2026-02-22T23:20:10.075534+0000","last_undegraded":"2026-02-22T23:20:10.075534+0000","last_fullsized":"2026-02-22T23:20:10.075534+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:36.205745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033533400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1115,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.063733+0000","last_change":"2026-02-22T22:56:20.934279+0000","last_active":"2026-02-22T23:20:10.063733+0000","last_peered":"2026-02-22T23:20:10.063733+0000","last_clean":"2026-02-22T23:20:10.063733+0000","last_became_active":"2026-02-22T22:56:18.931925+0000","last_became_peered":"2026-02-22T22:56:18.931925+0000","last_unstale":"2026-02-22T23:20:10.063733+0000","last_undegraded":"2026-02-22T23:20:10.063733+0000","last_fullsized":"2026-02-22T23:20:10.063733+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:28:58.350535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000296552,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1206,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.060915+0000","last_change":"2026-02-22T22:56:14.922932+0000","last_active":"2026-02-22T23:20:10.060915+0000","last_peered":"2026-02-22T23:20:10.060915+0000","last_clean":"2026-02-22T23:20:10.060915+0000","last_became_active":"2026-02-22T22:56:14.921193+0000","last_became_peered":"2026-02-22T22:56:14.921193+0000","last_unstale":"2026-02-22T23:20:10.060915+0000","last_undegraded":"2026-02-22T23:20:10.060915+0000","last_fullsized":"2026-02-22T23:20:10.060915+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_clean_scrub_stamp":"2026-02-22T22:56:13.916808+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:23:02.729111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705959,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89852,"kb_used_data":4584,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94281988,"statfs":{"total":96636764160,"available":96544755712,"internally_reserved":0,"allocated":4694016,"data_stored":2466938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":87292359},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705959,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":108484,"kb_used_data":5168,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":94263356,"statfs":{"total":96636764160,"available":96525676544,"internally_reserved":0,"allocated":5292032,"data_stored":3057325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":105773511},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738664,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":116804,"kb_used_data":5168,"kb_used_omap":1,"kb_used_meta":111614,"kb_avail":94255036,"statfs":{"total":96636764160,"available":96517156864,"internally_reserved":0,"allocated":5292032,"data_stored":3057325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114293190},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T23:20:36.605 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T23:20:36.605 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T23:20:36.761 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T23:20:36.761 INFO:teuthology.orchestra.run.trial060.stderr:dumped all 2026-02-22T23:20:36.769 INFO:teuthology.orchestra.run.trial060.stdout:{"pg_ready":true,"pg_map":{"version":1288,"stamp":"2026-02-22T23:20:35.418162+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":315140,"kb_used_data":14920,"kb_used_omap":4,"kb_used_meta":300155,"kb_avail":282800380,"statfs":{"total":289910292480,"available":289587589120,"internally_reserved":0,"allocated":15278080,"data_stored":8581588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":307359060},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000922"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.062791+0000","last_change":"2026-02-22T22:56:20.934556+0000","last_active":"2026-02-22T23:20:10.062791+0000","last_peered":"2026-02-22T23:20:10.062791+0000","last_clean":"2026-02-22T23:20:10.062791+0000","last_became_active":"2026-02-22T22:56:18.932408+0000","last_became_peered":"2026-02-22T22:56:18.932408+0000","last_unstale":"2026-02-22T23:20:10.062791+0000","last_undegraded":"2026-02-22T23:20:10.062791+0000","last_fullsized":"2026-02-22T23:20:10.062791+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:45:37.059532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052600900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.064272+0000","last_change":"2026-02-22T22:56:20.934743+0000","last_active":"2026-02-22T23:20:10.064272+0000","last_peered":"2026-02-22T23:20:10.064272+0000","last_clean":"2026-02-22T23:20:10.064272+0000","last_became_active":"2026-02-22T22:56:18.931993+0000","last_became_peered":"2026-02-22T22:56:18.931993+0000","last_unstale":"2026-02-22T23:20:10.064272+0000","last_undegraded":"2026-02-22T23:20:10.064272+0000","last_fullsized":"2026-02-22T23:20:10.064272+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:55.729675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071762499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.063967+0000","last_change":"2026-02-22T22:56:20.934280+0000","last_active":"2026-02-22T23:20:10.063967+0000","last_peered":"2026-02-22T23:20:10.063967+0000","last_clean":"2026-02-22T23:20:10.063967+0000","last_became_active":"2026-02-22T22:56:18.932505+0000","last_became_peered":"2026-02-22T22:56:18.932505+0000","last_unstale":"2026-02-22T23:20:10.063967+0000","last_undegraded":"2026-02-22T23:20:10.063967+0000","last_fullsized":"2026-02-22T23:20:10.063967+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:50:12.129550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000270864,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.062903+0000","last_change":"2026-02-22T22:56:20.934550+0000","last_active":"2026-02-22T23:20:10.062903+0000","last_peered":"2026-02-22T23:20:10.062903+0000","last_clean":"2026-02-22T23:20:10.062903+0000","last_became_active":"2026-02-22T22:56:18.932003+0000","last_became_peered":"2026-02-22T22:56:18.932003+0000","last_unstale":"2026-02-22T23:20:10.062903+0000","last_undegraded":"2026-02-22T23:20:10.062903+0000","last_fullsized":"2026-02-22T23:20:10.062903+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:26:14.433190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054626700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1116,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.516188+0000","last_change":"2026-02-22T22:56:20.932755+0000","last_active":"2026-02-22T23:20:10.516188+0000","last_peered":"2026-02-22T23:20:10.516188+0000","last_clean":"2026-02-22T23:20:10.516188+0000","last_became_active":"2026-02-22T22:56:18.932278+0000","last_became_peered":"2026-02-22T22:56:18.932278+0000","last_unstale":"2026-02-22T23:20:10.516188+0000","last_undegraded":"2026-02-22T23:20:10.516188+0000","last_fullsized":"2026-02-22T23:20:10.516188+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:54:51.140871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066678099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1113,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.076906+0000","last_change":"2026-02-22T22:56:21.006101+0000","last_active":"2026-02-22T23:20:10.076906+0000","last_peered":"2026-02-22T23:20:10.076906+0000","last_clean":"2026-02-22T23:20:10.076906+0000","last_became_active":"2026-02-22T22:56:18.932279+0000","last_became_peered":"2026-02-22T22:56:18.932279+0000","last_unstale":"2026-02-22T23:20:10.076906+0000","last_undegraded":"2026-02-22T23:20:10.076906+0000","last_fullsized":"2026-02-22T23:20:10.076906+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:12:56.667350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023402500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.075534+0000","last_change":"2026-02-22T22:56:21.006207+0000","last_active":"2026-02-22T23:20:10.075534+0000","last_peered":"2026-02-22T23:20:10.075534+0000","last_clean":"2026-02-22T23:20:10.075534+0000","last_became_active":"2026-02-22T22:56:18.931984+0000","last_became_peered":"2026-02-22T22:56:18.931984+0000","last_unstale":"2026-02-22T23:20:10.075534+0000","last_undegraded":"2026-02-22T23:20:10.075534+0000","last_fullsized":"2026-02-22T23:20:10.075534+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:36.205745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033533400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1115,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.063733+0000","last_change":"2026-02-22T22:56:20.934279+0000","last_active":"2026-02-22T23:20:10.063733+0000","last_peered":"2026-02-22T23:20:10.063733+0000","last_clean":"2026-02-22T23:20:10.063733+0000","last_became_active":"2026-02-22T22:56:18.931925+0000","last_became_peered":"2026-02-22T22:56:18.931925+0000","last_unstale":"2026-02-22T23:20:10.063733+0000","last_undegraded":"2026-02-22T23:20:10.063733+0000","last_fullsized":"2026-02-22T23:20:10.063733+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:28:58.350535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000296552,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1206,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.060915+0000","last_change":"2026-02-22T22:56:14.922932+0000","last_active":"2026-02-22T23:20:10.060915+0000","last_peered":"2026-02-22T23:20:10.060915+0000","last_clean":"2026-02-22T23:20:10.060915+0000","last_became_active":"2026-02-22T22:56:14.921193+0000","last_became_peered":"2026-02-22T22:56:14.921193+0000","last_unstale":"2026-02-22T23:20:10.060915+0000","last_undegraded":"2026-02-22T23:20:10.060915+0000","last_fullsized":"2026-02-22T23:20:10.060915+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_clean_scrub_stamp":"2026-02-22T22:56:13.916808+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:23:02.729111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705959,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89852,"kb_used_data":4584,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94281988,"statfs":{"total":96636764160,"available":96544755712,"internally_reserved":0,"allocated":4694016,"data_stored":2466938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":87292359},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705959,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":108484,"kb_used_data":5168,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":94263356,"statfs":{"total":96636764160,"available":96525676544,"internally_reserved":0,"allocated":5292032,"data_stored":3057325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":105773511},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738664,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":116804,"kb_used_data":5168,"kb_used_omap":1,"kb_used_meta":111614,"kb_avail":94255036,"statfs":{"total":96636764160,"available":96517156864,"internally_reserved":0,"allocated":5292032,"data_stored":3057325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114293190},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T23:20:36.770 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T23:20:36.928 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T23:20:36.928 INFO:teuthology.orchestra.run.trial060.stdout:{"epoch":558,"fsid":"c7d96b99-86c6-4a4e-9557-db334b970cfe","created":"2026-02-22T22:56:02.902367+0000","modified":"2026-02-22T23:20:10.057925+0000","last_up_change":"2026-02-22T22:56:12.915552+0000","last_in_change":"2026-02-22T22:56:03.466085+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:56:13.226544+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:56:17.081581+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f255f4c8-739d-4bd1-b589-bc63ac762e21","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6802","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6803","nonce":1437422725}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6804","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6805","nonce":1437422725}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6808","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6809","nonce":1437422725}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6806","nonce":1437422725},{"type":"v1","addr":"10.20.193.60:6807","nonce":1437422725}]},"public_addr":"10.20.193.60:6803/1437422725","cluster_addr":"10.20.193.60:6805/1437422725","heartbeat_back_addr":"10.20.193.60:6809/1437422725","heartbeat_front_addr":"10.20.193.60:6807/1437422725","state":["exists","up"]},{"osd":1,"uuid":"6ebac1ed-ac24-4619-94c0-c4cfe9c29d08","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":19,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6818","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6819","nonce":3794669997}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6820","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6821","nonce":3794669997}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6824","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6825","nonce":3794669997}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6822","nonce":3794669997},{"type":"v1","addr":"10.20.193.60:6823","nonce":3794669997}]},"public_addr":"10.20.193.60:6819/3794669997","cluster_addr":"10.20.193.60:6821/3794669997","heartbeat_back_addr":"10.20.193.60:6825/3794669997","heartbeat_front_addr":"10.20.193.60:6823/3794669997","state":["exists","up"]},{"osd":2,"uuid":"30a74eef-60f3-466e-8be5-b6e89425fb11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6810","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6811","nonce":4208307478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6812","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6813","nonce":4208307478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6816","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6817","nonce":4208307478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.60:6814","nonce":4208307478},{"type":"v1","addr":"10.20.193.60:6815","nonce":4208307478}]},"public_addr":"10.20.193.60:6811/4208307478","cluster_addr":"10.20.193.60:6813/4208307478","heartbeat_back_addr":"10.20.193.60:6817/4208307478","heartbeat_front_addr":"10.20.193.60:6815/4208307478","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.840981+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.942404+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:56:10.876837+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"10.20.193.82:0/2607606760":"2026-02-23T00:19:49.075302+0000","10.20.193.82:0/1451752147":"2026-02-23T00:19:49.075768+0000","10.20.193.82:0/817831085":"2026-02-23T00:19:49.075638+0000","10.20.193.82:0/2485920011":"2026-02-23T00:17:14.708397+0000","10.20.193.82:0/2849845567":"2026-02-23T00:19:18.444934+0000","10.20.193.82:0/57223455":"2026-02-23T00:17:14.926618+0000","10.20.193.82:0/48980392":"2026-02-23T00:17:45.047231+0000","10.20.193.82:0/2579076631":"2026-02-23T00:19:49.075912+0000","10.20.193.82:0/226477878":"2026-02-23T00:17:45.045335+0000","10.20.193.82:0/3729730171":"2026-02-23T00:17:45.045122+0000","10.20.193.82:0/2947085258":"2026-02-23T00:19:12.159032+0000","10.20.193.82:0/1127756589":"2026-02-23T00:19:49.075422+0000","10.20.193.82:0/758339745":"2026-02-23T00:17:45.047135+0000","10.20.193.82:0/3522464731":"2026-02-23T00:17:45.047331+0000","10.20.193.82:0/1674878184":"2026-02-23T00:17:45.045268+0000","10.20.193.82:0/441587789":"2026-02-23T00:19:49.075173+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":209,"length":1},{"begin":212,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T23:20:37.936 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-22T23:20:37.937 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T23:20:38.018 INFO:teuthology.orchestra.run.trial060.stdout:{ 2026-02-22T23:20:38.018 INFO:teuthology.orchestra.run.trial060.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T23:20:38.018 INFO:teuthology.orchestra.run.trial060.stdout:} 2026-02-22T23:20:38.023 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-22T23:20:38.178 INFO:teuthology.orchestra.run.trial060.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T23:20:38.188 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-22T23:20:38.188 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T23:20:38.270 INFO:teuthology.orchestra.run.trial060.stdout:{ 2026-02-22T23:20:38.270 INFO:teuthology.orchestra.run.trial060.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T23:20:38.270 INFO:teuthology.orchestra.run.trial060.stdout:} 2026-02-22T23:20:38.277 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-22T23:20:38.433 INFO:teuthology.orchestra.run.trial060.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T23:20:38.442 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-22T23:20:38.442 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T23:20:38.521 INFO:teuthology.orchestra.run.trial060.stdout:{ 2026-02-22T23:20:38.522 INFO:teuthology.orchestra.run.trial060.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T23:20:38.522 INFO:teuthology.orchestra.run.trial060.stdout:} 2026-02-22T23:20:38.527 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-22T23:20:38.685 INFO:teuthology.orchestra.run.trial060.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T23:20:38.694 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T23:20:38.852 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T23:20:38.853 INFO:teuthology.orchestra.run.trial060.stderr:dumped all 2026-02-22T23:20:38.859 INFO:teuthology.orchestra.run.trial060.stdout:{"pg_ready":true,"pg_map":{"version":1289,"stamp":"2026-02-22T23:20:37.418342+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":315140,"kb_used_data":14920,"kb_used_omap":4,"kb_used_meta":300155,"kb_avail":282800380,"statfs":{"total":289910292480,"available":289587589120,"internally_reserved":0,"allocated":15278080,"data_stored":8581588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":307359060},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000957"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.062791+0000","last_change":"2026-02-22T22:56:20.934556+0000","last_active":"2026-02-22T23:20:10.062791+0000","last_peered":"2026-02-22T23:20:10.062791+0000","last_clean":"2026-02-22T23:20:10.062791+0000","last_became_active":"2026-02-22T22:56:18.932408+0000","last_became_peered":"2026-02-22T22:56:18.932408+0000","last_unstale":"2026-02-22T23:20:10.062791+0000","last_undegraded":"2026-02-22T23:20:10.062791+0000","last_fullsized":"2026-02-22T23:20:10.062791+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:45:37.059532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052600900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.064272+0000","last_change":"2026-02-22T22:56:20.934743+0000","last_active":"2026-02-22T23:20:10.064272+0000","last_peered":"2026-02-22T23:20:10.064272+0000","last_clean":"2026-02-22T23:20:10.064272+0000","last_became_active":"2026-02-22T22:56:18.931993+0000","last_became_peered":"2026-02-22T22:56:18.931993+0000","last_unstale":"2026-02-22T23:20:10.064272+0000","last_undegraded":"2026-02-22T23:20:10.064272+0000","last_fullsized":"2026-02-22T23:20:10.064272+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:55.729675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071762499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.063967+0000","last_change":"2026-02-22T22:56:20.934280+0000","last_active":"2026-02-22T23:20:10.063967+0000","last_peered":"2026-02-22T23:20:10.063967+0000","last_clean":"2026-02-22T23:20:10.063967+0000","last_became_active":"2026-02-22T22:56:18.932505+0000","last_became_peered":"2026-02-22T22:56:18.932505+0000","last_unstale":"2026-02-22T23:20:10.063967+0000","last_undegraded":"2026-02-22T23:20:10.063967+0000","last_fullsized":"2026-02-22T23:20:10.063967+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:50:12.129550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000270864,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.062903+0000","last_change":"2026-02-22T22:56:20.934550+0000","last_active":"2026-02-22T23:20:10.062903+0000","last_peered":"2026-02-22T23:20:10.062903+0000","last_clean":"2026-02-22T23:20:10.062903+0000","last_became_active":"2026-02-22T22:56:18.932003+0000","last_became_peered":"2026-02-22T22:56:18.932003+0000","last_unstale":"2026-02-22T23:20:10.062903+0000","last_undegraded":"2026-02-22T23:20:10.062903+0000","last_fullsized":"2026-02-22T23:20:10.062903+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:26:14.433190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054626700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1116,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.516188+0000","last_change":"2026-02-22T22:56:20.932755+0000","last_active":"2026-02-22T23:20:10.516188+0000","last_peered":"2026-02-22T23:20:10.516188+0000","last_clean":"2026-02-22T23:20:10.516188+0000","last_became_active":"2026-02-22T22:56:18.932278+0000","last_became_peered":"2026-02-22T22:56:18.932278+0000","last_unstale":"2026-02-22T23:20:10.516188+0000","last_undegraded":"2026-02-22T23:20:10.516188+0000","last_fullsized":"2026-02-22T23:20:10.516188+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:54:51.140871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066678099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1113,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.076906+0000","last_change":"2026-02-22T22:56:21.006101+0000","last_active":"2026-02-22T23:20:10.076906+0000","last_peered":"2026-02-22T23:20:10.076906+0000","last_clean":"2026-02-22T23:20:10.076906+0000","last_became_active":"2026-02-22T22:56:18.932279+0000","last_became_peered":"2026-02-22T22:56:18.932279+0000","last_unstale":"2026-02-22T23:20:10.076906+0000","last_undegraded":"2026-02-22T23:20:10.076906+0000","last_fullsized":"2026-02-22T23:20:10.076906+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:12:56.667350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023402500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1114,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.075534+0000","last_change":"2026-02-22T22:56:21.006207+0000","last_active":"2026-02-22T23:20:10.075534+0000","last_peered":"2026-02-22T23:20:10.075534+0000","last_clean":"2026-02-22T23:20:10.075534+0000","last_became_active":"2026-02-22T22:56:18.931984+0000","last_became_peered":"2026-02-22T22:56:18.931984+0000","last_unstale":"2026-02-22T23:20:10.075534+0000","last_undegraded":"2026-02-22T23:20:10.075534+0000","last_fullsized":"2026-02-22T23:20:10.075534+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:36.205745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033533400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1115,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.063733+0000","last_change":"2026-02-22T22:56:20.934279+0000","last_active":"2026-02-22T23:20:10.063733+0000","last_peered":"2026-02-22T23:20:10.063733+0000","last_clean":"2026-02-22T23:20:10.063733+0000","last_became_active":"2026-02-22T22:56:18.931925+0000","last_became_peered":"2026-02-22T22:56:18.931925+0000","last_unstale":"2026-02-22T23:20:10.063733+0000","last_undegraded":"2026-02-22T23:20:10.063733+0000","last_fullsized":"2026-02-22T23:20:10.063733+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:17.923258+0000","last_clean_scrub_stamp":"2026-02-22T22:56:17.923258+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:28:58.350535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000296552,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1206,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:10.060915+0000","last_change":"2026-02-22T22:56:14.922932+0000","last_active":"2026-02-22T23:20:10.060915+0000","last_peered":"2026-02-22T23:20:10.060915+0000","last_clean":"2026-02-22T23:20:10.060915+0000","last_became_active":"2026-02-22T22:56:14.921193+0000","last_became_peered":"2026-02-22T22:56:14.921193+0000","last_unstale":"2026-02-22T23:20:10.060915+0000","last_undegraded":"2026-02-22T23:20:10.060915+0000","last_fullsized":"2026-02-22T23:20:10.060915+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:56:13.916808+0000","last_clean_scrub_stamp":"2026-02-22T22:56:13.916808+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:23:02.729111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705959,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89852,"kb_used_data":4584,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94281988,"statfs":{"total":96636764160,"available":96544755712,"internally_reserved":0,"allocated":4694016,"data_stored":2466938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":87292359},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705959,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":108484,"kb_used_data":5168,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":94263356,"statfs":{"total":96636764160,"available":96525676544,"internally_reserved":0,"allocated":5292032,"data_stored":3057325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":105773511},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738664,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":116804,"kb_used_data":5168,"kb_used_omap":1,"kb_used_meta":111614,"kb_avail":94255036,"statfs":{"total":96636764160,"available":96517156864,"internally_reserved":0,"allocated":5292032,"data_stored":3057325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114293190},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T23:20:38.861 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T22:56:17.923258+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=17, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=36, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:38.861 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T22:56:17.923258+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=17, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=36, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:38.862 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T22:56:17.923258+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=17, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=36, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:38.862 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T22:56:17.923258+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=17, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=36, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:38.862 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T22:56:17.923258+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=17, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=36, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:38.863 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T22:56:17.923258+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=17, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=36, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:38.863 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T22:56:17.923258+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=17, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=36, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:38.863 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T22:56:17.923258+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=17, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=36, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:38.863 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T22:56:13.916808+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=56, tm_sec=13, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=23, tm_min=20, tm_sec=36, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T23:20:38.864 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T23:20:58.865 DEBUG:teuthology.orchestra.run.trial060:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T23:20:59.023 INFO:teuthology.orchestra.run.trial060.stdout: 2026-02-22T23:20:59.024 INFO:teuthology.orchestra.run.trial060.stderr:dumped all 2026-02-22T23:20:59.032 INFO:teuthology.orchestra.run.trial060.stdout:{"pg_ready":true,"pg_map":{"version":1299,"stamp":"2026-02-22T23:20:57.419946+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":315128,"kb_used_data":14920,"kb_used_omap":4,"kb_used_meta":300155,"kb_avail":282800392,"statfs":{"total":289910292480,"available":289587601408,"internally_reserved":0,"allocated":15278080,"data_stored":8581588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":307359060},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000757"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1123,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:42.734912+0000","last_change":"2026-02-22T23:20:42.734924+0000","last_active":"2026-02-22T23:20:42.734912+0000","last_peered":"2026-02-22T23:20:42.734912+0000","last_clean":"2026-02-22T23:20:42.734912+0000","last_became_active":"2026-02-22T22:56:18.932408+0000","last_became_peered":"2026-02-22T22:56:18.932408+0000","last_unstale":"2026-02-22T23:20:42.734912+0000","last_undegraded":"2026-02-22T23:20:42.734912+0000","last_fullsized":"2026-02-22T23:20:42.734912+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:42.734859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:42.734859+0000","last_clean_scrub_stamp":"2026-02-22T23:20:42.734859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:59:09.441384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052600900000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1123,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:39.739493+0000","last_change":"2026-02-22T23:20:39.739508+0000","last_active":"2026-02-22T23:20:39.739493+0000","last_peered":"2026-02-22T23:20:39.739493+0000","last_clean":"2026-02-22T23:20:39.739493+0000","last_became_active":"2026-02-22T22:56:18.931993+0000","last_became_peered":"2026-02-22T22:56:18.931993+0000","last_unstale":"2026-02-22T23:20:39.739493+0000","last_undegraded":"2026-02-22T23:20:39.739493+0000","last_fullsized":"2026-02-22T23:20:39.739493+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:39.739443+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:39.739443+0000","last_clean_scrub_stamp":"2026-02-22T23:20:39.739443+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:54:45.136543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071762499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1123,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:43.712236+0000","last_change":"2026-02-22T23:20:43.712254+0000","last_active":"2026-02-22T23:20:43.712236+0000","last_peered":"2026-02-22T23:20:43.712236+0000","last_clean":"2026-02-22T23:20:43.712236+0000","last_became_active":"2026-02-22T22:56:18.932505+0000","last_became_peered":"2026-02-22T22:56:18.932505+0000","last_unstale":"2026-02-22T23:20:43.712236+0000","last_undegraded":"2026-02-22T23:20:43.712236+0000","last_fullsized":"2026-02-22T23:20:43.712236+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:43.712176+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:43.712176+0000","last_clean_scrub_stamp":"2026-02-22T23:20:43.712176+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:22:11.778328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000270864,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1123,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:40.729559+0000","last_change":"2026-02-22T23:20:40.729567+0000","last_active":"2026-02-22T23:20:40.729559+0000","last_peered":"2026-02-22T23:20:40.729559+0000","last_clean":"2026-02-22T23:20:40.729559+0000","last_became_active":"2026-02-22T22:56:18.932003+0000","last_became_peered":"2026-02-22T22:56:18.932003+0000","last_unstale":"2026-02-22T23:20:40.729559+0000","last_undegraded":"2026-02-22T23:20:40.729559+0000","last_fullsized":"2026-02-22T23:20:40.729559+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:40.729524+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:40.729524+0000","last_clean_scrub_stamp":"2026-02-22T23:20:40.729524+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:23:12.321696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054626700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"558'3","reported_seq":1126,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:38.789449+0000","last_change":"2026-02-22T23:20:38.789473+0000","last_active":"2026-02-22T23:20:38.789449+0000","last_peered":"2026-02-22T23:20:38.789449+0000","last_clean":"2026-02-22T23:20:38.789449+0000","last_became_active":"2026-02-22T22:56:18.932278+0000","last_became_peered":"2026-02-22T22:56:18.932278+0000","last_unstale":"2026-02-22T23:20:38.789449+0000","last_undegraded":"2026-02-22T23:20:38.789449+0000","last_fullsized":"2026-02-22T23:20:38.789449+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"558'3","last_scrub_stamp":"2026-02-22T23:20:38.789395+0000","last_deep_scrub":"558'3","last_deep_scrub_stamp":"2026-02-22T23:20:38.789395+0000","last_clean_scrub_stamp":"2026-02-22T23:20:38.789395+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:15:36.959336+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00066678099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1122,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:39.902588+0000","last_change":"2026-02-22T23:20:39.902606+0000","last_active":"2026-02-22T23:20:39.902588+0000","last_peered":"2026-02-22T23:20:39.902588+0000","last_clean":"2026-02-22T23:20:39.902588+0000","last_became_active":"2026-02-22T22:56:18.932279+0000","last_became_peered":"2026-02-22T22:56:18.932279+0000","last_unstale":"2026-02-22T23:20:39.902588+0000","last_undegraded":"2026-02-22T23:20:39.902588+0000","last_fullsized":"2026-02-22T23:20:39.902588+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:39.902542+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:39.902542+0000","last_clean_scrub_stamp":"2026-02-22T23:20:39.902542+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:37:52.684013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023402500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1123,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:38.922493+0000","last_change":"2026-02-22T23:20:38.922506+0000","last_active":"2026-02-22T23:20:38.922493+0000","last_peered":"2026-02-22T23:20:38.922493+0000","last_clean":"2026-02-22T23:20:38.922493+0000","last_became_active":"2026-02-22T22:56:18.931984+0000","last_became_peered":"2026-02-22T22:56:18.931984+0000","last_unstale":"2026-02-22T23:20:38.922493+0000","last_undegraded":"2026-02-22T23:20:38.922493+0000","last_fullsized":"2026-02-22T23:20:38.922493+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T23:20:38.922442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T23:20:38.922442+0000","last_clean_scrub_stamp":"2026-02-22T23:20:38.922442+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:12:53.712137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033533400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"558'2","reported_seq":1125,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:41.743912+0000","last_change":"2026-02-22T23:20:41.743937+0000","last_active":"2026-02-22T23:20:41.743912+0000","last_peered":"2026-02-22T23:20:41.743912+0000","last_clean":"2026-02-22T23:20:41.743912+0000","last_became_active":"2026-02-22T22:56:18.931925+0000","last_became_peered":"2026-02-22T22:56:18.931925+0000","last_unstale":"2026-02-22T23:20:41.743912+0000","last_undegraded":"2026-02-22T23:20:41.743912+0000","last_fullsized":"2026-02-22T23:20:41.743912+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"558'2","last_scrub_stamp":"2026-02-22T23:20:41.743821+0000","last_deep_scrub":"558'2","last_deep_scrub_stamp":"2026-02-22T23:20:41.743821+0000","last_clean_scrub_stamp":"2026-02-22T23:20:41.743821+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:15:54.346438+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.000296552,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"558'74","reported_seq":1217,"reported_epoch":558,"state":"active+clean","last_fresh":"2026-02-22T23:20:38.732907+0000","last_change":"2026-02-22T23:20:38.732926+0000","last_active":"2026-02-22T23:20:38.732907+0000","last_peered":"2026-02-22T23:20:38.732907+0000","last_clean":"2026-02-22T23:20:38.732907+0000","last_became_active":"2026-02-22T22:56:14.921193+0000","last_became_peered":"2026-02-22T22:56:14.921193+0000","last_unstale":"2026-02-22T23:20:38.732907+0000","last_undegraded":"2026-02-22T23:20:38.732907+0000","last_fullsized":"2026-02-22T23:20:38.732907+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"558'74","last_scrub_stamp":"2026-02-22T23:20:38.732872+0000","last_deep_scrub":"558'74","last_deep_scrub_stamp":"2026-02-22T23:20:38.732872+0000","last_clean_scrub_stamp":"2026-02-22T23:20:38.732872+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:22:26.536755+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705963,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":89848,"kb_used_data":4584,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94281992,"statfs":{"total":96636764160,"available":96544759808,"internally_reserved":0,"allocated":4694016,"data_stored":2466938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":87292359},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705963,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":108480,"kb_used_data":5168,"kb_used_omap":1,"kb_used_meta":103294,"kb_avail":94263360,"statfs":{"total":96636764160,"available":96525680640,"internally_reserved":0,"allocated":5292032,"data_stored":3057325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":105773511},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738668,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":116800,"kb_used_data":5168,"kb_used_omap":1,"kb_used_meta":111614,"kb_avail":94255040,"statfs":{"total":96636764160,"available":96517160960,"internally_reserved":0,"allocated":5292032,"data_stored":3057325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":114293190},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T23:20:59.034 DEBUG:teuthology.orchestra.run.trial060:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-22T23:20:59.198 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T23:20:59.198 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T23:20:59.198 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-22T23:20:59.198 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T23:20:59.234 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-22T23:20:59.235 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-22T23:20:59.235 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T23:20:59.313 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-22T23:20:59.313 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-22T23:20:59.314 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T23:20:59.391 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-22T23:20:59.392 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T23:20:59.392 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-22T23:20:59.392 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T23:20:59.467 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-22T23:20:59.467 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T23:20:59.467 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-22T23:20:59.468 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T23:20:59.518 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-22T23:20:59.519 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T23:20:59.519 DEBUG:teuthology.orchestra.run.trial060:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T23:20:59.584 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial060.front.sepia.ceph.com 2026-02-22T23:20:59.585 DEBUG:teuthology.orchestra.run.trial060:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-22T23:20:59.658 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial060.front.sepia.ceph.com 2026-02-22T23:20:59.658 DEBUG:teuthology.orchestra.run.trial060:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-22T23:20:59.728 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial060.front.sepia.ceph.com 2026-02-22T23:20:59.729 DEBUG:teuthology.orchestra.run.trial060:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-22T23:20:59.814 INFO:tasks.ceph:Archiving mon data... 2026-02-22T23:20:59.815 DEBUG:teuthology.misc:Transferring archived files from trial060:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284/data/cluster1.mon.a.tgz 2026-02-22T23:20:59.815 DEBUG:teuthology.orchestra.run.trial060:> mktemp 2026-02-22T23:20:59.872 INFO:teuthology.orchestra.run.trial060.stdout:/tmp/tmp.Cd8NWuJKHR 2026-02-22T23:20:59.872 DEBUG:teuthology.orchestra.run.trial060:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.Cd8NWuJKHR 2026-02-22T23:21:00.009 DEBUG:teuthology.orchestra.run.trial060:> sudo chmod 0666 /tmp/tmp.Cd8NWuJKHR 2026-02-22T23:21:00.092 DEBUG:teuthology.orchestra.remote:trial060:/tmp/tmp.Cd8NWuJKHR is 575KB 2026-02-22T23:21:00.155 DEBUG:teuthology.orchestra.run.trial060:> rm -fr /tmp/tmp.Cd8NWuJKHR 2026-02-22T23:21:00.172 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T23:21:00.172 DEBUG:teuthology.orchestra.run.trial060:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T23:21:00.213 DEBUG:teuthology.orchestra.run.trial082:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T23:21:00.291 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T23:21:00.307 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T23:21:00.307 INFO:tasks.ceph:Archiving crash dumps... 2026-02-22T23:21:00.310 DEBUG:teuthology.misc:Transferring archived files from trial060:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284/remote/trial060/crash 2026-02-22T23:21:00.311 DEBUG:teuthology.orchestra.run.trial060:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T23:21:00.357 DEBUG:teuthology.misc:Transferring archived files from trial082:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284/remote/trial082/crash 2026-02-22T23:21:00.359 DEBUG:teuthology.orchestra.run.trial082:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T23:21:00.382 INFO:tasks.ceph:Compressing logs... 2026-02-22T23:21:00.382 DEBUG:teuthology.orchestra.run.trial060:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T23:21:00.403 DEBUG:teuthology.orchestra.run.trial082:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T23:21:00.425 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41768.log 2026-02-22T23:21:00.435 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41866.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.41768.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41768.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42419.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.41866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38574.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41866.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38668.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42419.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:gzip/var/log/ceph/cluster1-client.admin.38574.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.41703.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.38574.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.38668.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.38668.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41608.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.41703.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38863.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.41703.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.41608.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41608.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.38863.log: gzip -5 --verbose -- 0.0% /var/log/ceph/cluster1-client.admin.43556.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.38863.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43556.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 0.0% 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.43556.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43654.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43801.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43654.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43654.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38795.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43801.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43801.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43605.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.38795.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.38795.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43507.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43605.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41517.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43605.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43507.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43507.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42209.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.41517.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41517.log.gz 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42209.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38389.log 2026-02-22T23:21:00.436 INFO:teuthology.orchestra.run.trial060.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42209.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41678.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.38389.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43703.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.38389.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.41678.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41678.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42116.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43703.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43703.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43752.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42116.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42116.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43458.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43752.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43752.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38621.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43458.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43458.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42264.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.38621.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.38621.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41817.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42264.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42264.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43899.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.41817.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41817.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43899.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43899.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43946.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip/var/log/ceph/cluster1-mgr.x.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.43850.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43946.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43946.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.33487.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43850.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43360.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43850.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/ceph.tmp-client.admin.33487.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.33487.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42552.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42265.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43360.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43360.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42552.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42552.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42265.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41967.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42265.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42601.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.41967.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41967.log.gz 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42503.log 2026-02-22T23:21:00.437 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42601.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42601.log.gz 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41969.log 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42503.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42503.log.gz 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42114.log 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.41969.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41969.log.gz 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42411.log 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42114.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42114.log.gz 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43409.log 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42411.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42411.log.gz 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41968.log 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.43409.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43409.log.gz 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42115.log 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.41968.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41968.log.gz 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1-client.admin.42115.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42115.log.gz 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42356.log 2026-02-22T23:21:00.439 INFO:teuthology.orchestra.run.trial060.stderr:/var/log/ceph/cluster1.log: /var/log/ceph/cluster1-client.admin.42356.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42356.log.gz 2026-02-22T23:21:00.440 INFO:teuthology.orchestra.run.trial060.stderr: 87.0% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-22T23:21:00.440 INFO:teuthology.orchestra.run.trial060.stderr: 93.6% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-22T23:21:00.447 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.53991.log 2026-02-22T23:21:00.447 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73478.log 2026-02-22T23:21:00.448 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.2.53991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70061.log 2026-02-22T23:21:00.448 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73478.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72390.log 2026-02-22T23:21:00.448 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73478.log.gz 2026-02-22T23:21:00.448 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59932.log 2026-02-22T23:21:00.448 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70061.log: /var/log/ceph/cluster1-client.admin.72390.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48930.log 2026-02-22T23:21:00.449 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.70061.log.gz 2026-02-22T23:21:00.449 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.59932.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.72390.log.gz 2026-02-22T23:21:00.449 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55770.log 2026-02-22T23:21:00.449 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.59932.log.gz 2026-02-22T23:21:00.449 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.48930.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74143.log 2026-02-22T23:21:00.449 INFO:teuthology.orchestra.run.trial082.stderr: 57.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48930.log.gz 2026-02-22T23:21:00.449 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.55770.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52635.log 2026-02-22T23:21:00.450 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74143.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56035.log 2026-02-22T23:21:00.450 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.55770.log.gz 2026-02-22T23:21:00.450 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52635.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72537.log 2026-02-22T23:21:00.450 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.52635.log.gz 2026-02-22T23:21:00.450 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.56035.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.53931.log 2026-02-22T23:21:00.451 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72537.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56035.log.gz 2026-02-22T23:21:00.451 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64862.log 2026-02-22T23:21:00.451 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.74143.log.gz 2026-02-22T23:21:00.451 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.1.53931.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.72537.log.gz 2026-02-22T23:21:00.451 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57116.log 2026-02-22T23:21:00.451 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73688.log 2026-02-22T23:21:00.451 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.64862.log.gz 2026-02-22T23:21:00.451 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.57116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74959.log 2026-02-22T23:21:00.452 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73688.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85269.log 2026-02-22T23:21:00.452 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57116.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.73688.log.gz 2026-02-22T23:21:00.452 INFO:teuthology.orchestra.run.trial082.stderr: 2026-02-22T23:21:00.452 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74959.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46956.log 2026-02-22T23:21:00.452 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74959.log.gz 2026-02-22T23:21:00.452 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.85269.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88625.log 2026-02-22T23:21:00.452 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85269.log.gz 2026-02-22T23:21:00.452 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.46956.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.84078.log 2026-02-22T23:21:00.452 INFO:teuthology.orchestra.run.trial082.stderr: 80.4% -- replaced with /var/log/ceph/cluster2-client.admin.46956.log.gz 2026-02-22T23:21:00.453 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88625.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65453.log 2026-02-22T23:21:00.453 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88625.log.gz 2026-02-22T23:21:00.453 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.0.84078.log: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.84078.log.gz 2026-02-22T23:21:00.453 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59830.log 2026-02-22T23:21:00.453 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65453.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47222.log 2026-02-22T23:21:00.453 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.59830.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.65453.log.gz 2026-02-22T23:21:00.453 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49225.log 2026-02-22T23:21:00.453 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59830.log.gz 2026-02-22T23:21:00.454 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47222.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50027.log 2026-02-22T23:21:00.454 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47222.log.gz 2026-02-22T23:21:00.454 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49225.log: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49225.log.gz 2026-02-22T23:21:00.454 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63485.log 2026-02-22T23:21:00.454 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.50027.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82287.log 2026-02-22T23:21:00.454 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.63485.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.50027.log.gz 2026-02-22T23:21:00.454 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57401.log 2026-02-22T23:21:00.454 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63485.log.gz 2026-02-22T23:21:00.455 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82287.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54817.log 2026-02-22T23:21:00.455 INFO:teuthology.orchestra.run.trial082.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82287.log.gz 2026-02-22T23:21:00.455 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57401.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71411.log 2026-02-22T23:21:00.455 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.57401.log.gz 2026-02-22T23:21:00.455 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.54817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58855.log 2026-02-22T23:21:00.455 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.71411.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.54817.log.gz 2026-02-22T23:21:00.455 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68184.log 2026-02-22T23:21:00.456 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58855.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.71411.log.gz 2026-02-22T23:21:00.456 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84767.log 2026-02-22T23:21:00.456 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.58855.log.gz 2026-02-22T23:21:00.456 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68184.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69791.log 2026-02-22T23:21:00.456 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84767.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58484.log 2026-02-22T23:21:00.456 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.68184.log.gz 2026-02-22T23:21:00.457 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.69791.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53699.log 2026-02-22T23:21:00.457 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58484.log: gzip -5 --verbose 89.9% -- /var/log/ceph/cluster1-client.mirror.77644.log -- replaced with /var/log/ceph/cluster1-client.mirror.69791.log.gz 2026-02-22T23:21:00.457 INFO:teuthology.orchestra.run.trial082.stderr: 2026-02-22T23:21:00.457 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.58484.log.gz 2026-02-22T23:21:00.458 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53699.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75921.log 2026-02-22T23:21:00.459 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.77644.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53236.log 2026-02-22T23:21:00.459 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75921.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53699.log.gz 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58671.log 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77644.log.gz 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75921.log.gz 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.53236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48344.log 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53236.log.gz 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58671.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72985.log 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.48344.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.58671.log.gz 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48344.log.gz 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66297.log 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72985.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75368.log 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.66297.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84767.log.gz 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74938.log 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72985.log.gz 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.66297.log.gz 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.84089.log 2026-02-22T23:21:00.460 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75368.log: /var/log/ceph/cluster2-client.mirror.74938.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70600.log 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.75368.log.gz 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.1.84089.log: 19.0% -- replaced with /var/log/ceph/cluster2-client.mirror.1.84089.log.gz 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47489.log 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74938.log.gz 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.70600.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61953.log 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47489.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47489.log.gz 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81440.log 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.61953.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53626.log 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.2.53991.log.gz 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.70600.log.gz 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81440.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60572.log 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61953.log.gz 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60169.log 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53626.log.gz 2026-02-22T23:21:00.462 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.60572.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81440.log.gz 2026-02-22T23:21:00.463 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70248.log 2026-02-22T23:21:00.463 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.60169.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60572.log.gz 2026-02-22T23:21:00.463 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66836.log 2026-02-22T23:21:00.463 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.70248.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62657.log 2026-02-22T23:21:00.463 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70248.log.gz 2026-02-22T23:21:00.464 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60169.log.gz 2026-02-22T23:21:00.464 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71599.log 2026-02-22T23:21:00.464 INFO:teuthology.orchestra.run.trial082.stderr: 87.7%/var/log/ceph/cluster2-client.admin.62657.log: -- replaced with /var/log/ceph/cluster1-client.admin.66836.log.gz 2026-02-22T23:21:00.464 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80260.log 2026-02-22T23:21:00.464 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.62657.log.gz 2026-02-22T23:21:00.464 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.71599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47488.log 2026-02-22T23:21:00.465 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.80260.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.71599.log.gz 2026-02-22T23:21:00.465 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.85432.log 2026-02-22T23:21:00.465 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76359.log 2026-02-22T23:21:00.465 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47488.log.gz 2026-02-22T23:21:00.465 INFO:teuthology.orchestra.run.trial082.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.80260.log.gz 2026-02-22T23:21:00.465 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.85432.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74505.log 2026-02-22T23:21:00.465 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55009.log 2026-02-22T23:21:00.465 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76359.log.gz 2026-02-22T23:21:00.465 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74505.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77145.log 2026-02-22T23:21:00.465 INFO:teuthology.orchestra.run.trial082.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74505.log.gz 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.55009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54420.log 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.55009.log.gz 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.77145.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60297.log 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77145.log.gz 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72177.log 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.54420.log: /var/log/ceph/cluster2-client.admin.60297.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69074.log 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.72177.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.60297.log.gz 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68695.log 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54420.log.gz 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.72177.log.gz 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.69074.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72944.log 2026-02-22T23:21:00.466 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69074.log.gz 2026-02-22T23:21:00.467 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.68695.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58833.log 2026-02-22T23:21:00.467 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72944.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58104.log 2026-02-22T23:21:00.468 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58833.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50108.log 2026-02-22T23:21:00.468 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.68695.log.gz 2026-02-22T23:21:00.468 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58104.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.58833.log.gz 2026-02-22T23:21:00.468 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72944.log.gz 2026-02-22T23:21:00.468 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66029.log 2026-02-22T23:21:00.468 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.58104.log.gz 2026-02-22T23:21:00.468 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.50108.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49932.log 2026-02-22T23:21:00.468 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.50108.log.gz 2026-02-22T23:21:00.468 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.66029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68124.log 2026-02-22T23:21:00.468 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49932.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74094.log 2026-02-22T23:21:00.469 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.68124.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58752.log 2026-02-22T23:21:00.469 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.66029.log.gz 2026-02-22T23:21:00.469 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74094.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56629.log 2026-02-22T23:21:00.469 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68124.log.gz 2026-02-22T23:21:00.469 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58752.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86808.log 2026-02-22T23:21:00.469 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.56629.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.58752.log.gz 2026-02-22T23:21:00.469 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68238.log 2026-02-22T23:21:00.469 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.56629.log.gz 2026-02-22T23:21:00.469 INFO:teuthology.orchestra.run.trial082.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.74094.log.gz 2026-02-22T23:21:00.470 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.86808.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57261.log 2026-02-22T23:21:00.470 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68238.log: gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-22T23:21:00.470 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.86808.log.gz 2026-02-22T23:21:00.470 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.68238.log.gz 2026-02-22T23:21:00.470 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57261.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75004.log 2026-02-22T23:21:00.470 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.57261.log.gz 2026-02-22T23:21:00.470 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67338.log 2026-02-22T23:21:00.470 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75004.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.49430.log 2026-02-22T23:21:00.471 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49932.log.gz 2026-02-22T23:21:00.471 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.67338.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75004.log.gz 2026-02-22T23:21:00.471 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.53889.log 2026-02-22T23:21:00.471 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.49430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66650.log 2026-02-22T23:21:00.471 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67338.log.gz 2026-02-22T23:21:00.471 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.1.53889.log: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.1.53889.log.gz 2026-02-22T23:21:00.471 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73562.log 2026-02-22T23:21:00.471 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.66650.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57739.log 2026-02-22T23:21:00.471 INFO:teuthology.orchestra.run.trial082.stderr: 86.9% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-22T23:21:00.471 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73562.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.66650.log.gz 2026-02-22T23:21:00.472 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84053.log 2026-02-22T23:21:00.472 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57739.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57341.log 2026-02-22T23:21:00.472 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84053.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.57739.log.gz 2026-02-22T23:21:00.472 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68876.log 2026-02-22T23:21:00.472 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.57341.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74398.log 2026-02-22T23:21:00.472 INFO:teuthology.orchestra.run.trial082.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.73562.log.gz 2026-02-22T23:21:00.473 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.68876.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59668.log 2026-02-22T23:21:00.473 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.57341.log.gz 2026-02-22T23:21:00.473 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74398.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.83026.log 2026-02-22T23:21:00.473 INFO:teuthology.orchestra.run.trial082.stderr: 90.5%/var/log/ceph/cluster1-client.admin.59668.log: -- replaced with /var/log/ceph/cluster2-client.mirror.84053.log.gz 2026-02-22T23:21:00.473 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49054.log 2026-02-22T23:21:00.473 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.68876.log.gz 2026-02-22T23:21:00.473 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.59668.log.gz 2026-02-22T23:21:00.473 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74398.log.gz 2026-02-22T23:21:00.474 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.83026.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63614.log 2026-02-22T23:21:00.474 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.49054.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51902.log 2026-02-22T23:21:00.474 INFO:teuthology.orchestra.run.trial082.stderr: 56.3% -- replaced with /var/log/ceph/cluster1-client.admin.49054.log.gz 2026-02-22T23:21:00.474 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.63614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56736.log 2026-02-22T23:21:00.474 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.63614.log.gz 2026-02-22T23:21:00.474 INFO:teuthology.orchestra.run.trial082.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.1.53931.log.gz 2026-02-22T23:21:00.474 INFO:teuthology.orchestra.run.trial082.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.49430.log.gz 2026-02-22T23:21:00.474 INFO:teuthology.orchestra.run.trial082.stderr:gzip/var/log/ceph/cluster1-client.admin.51902.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49174.log 2026-02-22T23:21:00.475 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.56736.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.85467.log 2026-02-22T23:21:00.475 INFO:teuthology.orchestra.run.trial082.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.3.83026.log.gz 2026-02-22T23:21:00.475 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51902.log.gz 2026-02-22T23:21:00.475 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49174.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63766.log 2026-02-22T23:21:00.475 INFO:teuthology.orchestra.run.trial082.stderr: 56.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49174.log.gz 2026-02-22T23:21:00.475 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.85467.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56736.log.gz 2026-02-22T23:21:00.475 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58293.log 2026-02-22T23:21:00.475 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63766.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79608.log 2026-02-22T23:21:00.475 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.63766.log.gz 2026-02-22T23:21:00.475 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.58293.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56683.log 2026-02-22T23:21:00.476 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.58293.log.gz 2026-02-22T23:21:00.476 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.79608.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53606.log 2026-02-22T23:21:00.476 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.79608.log.gz 2026-02-22T23:21:00.476 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.56683.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70651.log 2026-02-22T23:21:00.476 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.56683.log.gz 2026-02-22T23:21:00.476 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53606.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80861.log 2026-02-22T23:21:00.476 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.70651.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69016.log 2026-02-22T23:21:00.477 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70651.log.gz 2026-02-22T23:21:00.477 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80861.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53606.log.gz 2026-02-22T23:21:00.477 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83550.log 2026-02-22T23:21:00.477 INFO:teuthology.orchestra.run.trial082.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80861.log.gz 2026-02-22T23:21:00.477 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.69016.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51875.log 2026-02-22T23:21:00.477 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.83550.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60383.log 2026-02-22T23:21:00.477 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69016.log.gz 2026-02-22T23:21:00.478 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51875.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83550.log.gz 2026-02-22T23:21:00.478 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66677.log 2026-02-22T23:21:00.478 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.51875.log.gz 2026-02-22T23:21:00.478 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60383.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49325.log 2026-02-22T23:21:00.478 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.66677.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.60383.log.gz 2026-02-22T23:21:00.478 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78115.log 2026-02-22T23:21:00.478 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.66677.log.gz 2026-02-22T23:21:00.479 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.49325.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.49325.log.gz 2026-02-22T23:21:00.479 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72268.log 2026-02-22T23:21:00.479 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.78115.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54452.log 2026-02-22T23:21:00.479 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72268.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.78115.log.gz 2026-02-22T23:21:00.479 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56869.log 2026-02-22T23:21:00.479 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.54452.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73966.log 2026-02-22T23:21:00.480 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.56869.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72268.log.gz 2026-02-22T23:21:00.480 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68990.log 2026-02-22T23:21:00.480 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.56869.log.gz 2026-02-22T23:21:00.480 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59351.log 2026-02-22T23:21:00.480 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73966.log.gz 2026-02-22T23:21:00.480 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81841.log 2026-02-22T23:21:00.481 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68990.log: /var/log/ceph/cluster1-client.mirror.59351.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.68990.log.gz 2026-02-22T23:21:00.481 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76005.log 2026-02-22T23:21:00.481 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50141.log/var/log/ceph/cluster1-client.mirror.81841.log: 2026-02-22T23:21:00.481 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59351.log.gz 2026-02-22T23:21:00.481 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.81841.log.gz 2026-02-22T23:21:00.481 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70164.log 2026-02-22T23:21:00.481 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.50141.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75025.log 88.3% 2026-02-22T23:21:00.481 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.76005.log.gz 2026-02-22T23:21:00.482 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.70164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79020.log 2026-02-22T23:21:00.482 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.70164.log.gz 2026-02-22T23:21:00.482 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75025.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66809.log 2026-02-22T23:21:00.482 INFO:teuthology.orchestra.run.trial082.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.50141.log.gz 2026-02-22T23:21:00.482 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.75025.log.gz 2026-02-22T23:21:00.482 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.79020.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57210.log 2026-02-22T23:21:00.482 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.54452.log.gz 2026-02-22T23:21:00.483 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66809.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71695.log 2026-02-22T23:21:00.483 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.79020.log.gz 2026-02-22T23:21:00.483 INFO:teuthology.orchestra.run.trial082.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.66809.log.gz 2026-02-22T23:21:00.483 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.57210.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81576.log 2026-02-22T23:21:00.483 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57210.log.gz 2026-02-22T23:21:00.483 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.71695.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75408.log 2026-02-22T23:21:00.483 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.71695.log.gz 2026-02-22T23:21:00.483 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81576.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57820.log 2026-02-22T23:21:00.483 INFO:teuthology.orchestra.run.trial082.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.81576.log.gz 2026-02-22T23:21:00.483 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75408.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74485.log 2026-02-22T23:21:00.484 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.57820.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53749.log 2026-02-22T23:21:00.484 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75408.log.gz 2026-02-22T23:21:00.484 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.57820.log.gz 2026-02-22T23:21:00.484 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74485.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47221.log 2026-02-22T23:21:00.484 INFO:teuthology.orchestra.run.trial082.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.74485.log.gz 2026-02-22T23:21:00.484 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.53749.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68610.log 2026-02-22T23:21:00.484 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47221.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53749.log.gz 2026-02-22T23:21:00.485 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.80303.log 2026-02-22T23:21:00.485 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47221.log.gz 2026-02-22T23:21:00.485 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68610.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60522.log 2026-02-22T23:21:00.485 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.68610.log.gz 2026-02-22T23:21:00.485 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.80303.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.80303.log.gz 2026-02-22T23:21:00.485 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47342.log 2026-02-22T23:21:00.485 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.60522.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73337.log 2026-02-22T23:21:00.485 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47342.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47342.log.gz 2026-02-22T23:21:00.485 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52885.log 2026-02-22T23:21:00.486 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73337.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60522.log.gz 2026-02-22T23:21:00.486 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71966.log 2026-02-22T23:21:00.486 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.52885.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66055.log 2026-02-22T23:21:00.486 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73337.log.gz 2026-02-22T23:21:00.486 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64415.log 2026-02-22T23:21:00.486 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.66055.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72030.log 2026-02-22T23:21:00.487 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71966.log.gz 2026-02-22T23:21:00.487 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66055.log.gz 2026-02-22T23:21:00.487 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.64415.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52885.log.gz 2026-02-22T23:21:00.487 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56140.log 2026-02-22T23:21:00.487 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.64415.log.gz 2026-02-22T23:21:00.487 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72030.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83795.log 2026-02-22T23:21:00.487 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.56140.log: 87.4%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.72030.log.gz -5 2026-02-22T23:21:00.487 INFO:teuthology.orchestra.run.trial082.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.78867.log 2026-02-22T23:21:00.488 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83795.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.56140.log.gz 2026-02-22T23:21:00.488 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.82861.log 2026-02-22T23:21:00.488 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.83795.log.gz 2026-02-22T23:21:00.488 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78867.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79745.log 2026-02-22T23:21:00.488 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.82861.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78867.log.gz 2026-02-22T23:21:00.488 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86336.log 2026-02-22T23:21:00.488 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.82861.log.gz 2026-02-22T23:21:00.488 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.79745.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58778.log 2026-02-22T23:21:00.489 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.86336.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.79745.log.gz 2026-02-22T23:21:00.489 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77767.log 2026-02-22T23:21:00.489 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.86336.log.gz 2026-02-22T23:21:00.489 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.58778.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79938.log 2026-02-22T23:21:00.489 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.77767.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.58778.log.gz 2026-02-22T23:21:00.489 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77922.log 2026-02-22T23:21:00.489 INFO:teuthology.orchestra.run.trial082.stderr: 87.2%/var/log/ceph/cluster2-client.admin.79938.log: -- replaced with /var/log/ceph/cluster2-client.admin.77767.log.gz 2026-02-22T23:21:00.490 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48393.log 2026-02-22T23:21:00.490 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.77922.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72366.log 2026-02-22T23:21:00.490 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.79938.log.gz 2026-02-22T23:21:00.490 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.48393.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48393.log.gz 2026-02-22T23:21:00.490 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61220.log 2026-02-22T23:21:00.490 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54703.log 2026-02-22T23:21:00.490 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72366.log: /var/log/ceph/cluster2-client.mirror.61220.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53848.log 2026-02-22T23:21:00.491 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.72366.log.gz 2026-02-22T23:21:00.491 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.54703.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61220.log.gz 2026-02-22T23:21:00.491 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64189.log 2026-02-22T23:21:00.491 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.53848.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67021.log 2026-02-22T23:21:00.491 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.53848.log.gz 2026-02-22T23:21:00.491 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.54703.log.gz 2026-02-22T23:21:00.491 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.64189.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81620.log 2026-02-22T23:21:00.492 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67021.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79884.log 2026-02-22T23:21:00.492 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.67021.log.gz 2026-02-22T23:21:00.492 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81620.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48099.log 2026-02-22T23:21:00.492 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64189.log.gz 2026-02-22T23:21:00.492 INFO:teuthology.orchestra.run.trial082.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81620.log.gz 2026-02-22T23:21:00.492 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.79884.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82809.log 2026-02-22T23:21:00.493 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.48099.log: 0.0% 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.48099.log.gz 2026-02-22T23:21:00.493 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.79884.log.gz 2026-02-22T23:21:00.493 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75102.log 2026-02-22T23:21:00.493 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82809.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66596.log 2026-02-22T23:21:00.493 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82809.log.gz 2026-02-22T23:21:00.493 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49020.log 2026-02-22T23:21:00.493 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66596.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76401.log 2026-02-22T23:21:00.494 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.66596.log.gz 2026-02-22T23:21:00.494 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77922.log.gz 2026-02-22T23:21:00.494 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.49020.log: 52.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49020.log.gz 2026-02-22T23:21:00.494 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73357.log 2026-02-22T23:21:00.494 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76401.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77488.log 2026-02-22T23:21:00.494 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73357.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76401.log.gz 2026-02-22T23:21:00.494 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58623.log 2026-02-22T23:21:00.495 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.77488.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52464.log 2026-02-22T23:21:00.495 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73357.log.gz 2026-02-22T23:21:00.495 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.58623.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.77488.log.gz 2026-02-22T23:21:00.495 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56371.log 2026-02-22T23:21:00.495 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52464.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58623.log.gz 2026-02-22T23:21:00.495 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76422.log 2026-02-22T23:21:00.495 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.52464.log.gz 2026-02-22T23:21:00.495 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.56371.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77350.log 2026-02-22T23:21:00.496 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76422.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50885.log 2026-02-22T23:21:00.496 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56371.log.gz 2026-02-22T23:21:00.496 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.77350.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75102.log.gz 2026-02-22T23:21:00.496 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65620.log 2026-02-22T23:21:00.496 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.77350.log.gz 2026-02-22T23:21:00.496 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.50885.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76446.log 2026-02-22T23:21:00.496 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.50885.log.gz 2026-02-22T23:21:00.496 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.65620.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83640.log 2026-02-22T23:21:00.496 INFO:teuthology.orchestra.run.trial082.stderr: 91.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76422.log.gz 2026-02-22T23:21:00.497 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76446.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78706.log 2026-02-22T23:21:00.497 INFO:teuthology.orchestra.run.trial082.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76446.log.gz 2026-02-22T23:21:00.497 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83640.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65620.log.gz 2026-02-22T23:21:00.497 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72557.log 2026-02-22T23:21:00.497 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.83640.log.gz 2026-02-22T23:21:00.497 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.78706.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67522.log 2026-02-22T23:21:00.497 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72557.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60925.log 2026-02-22T23:21:00.498 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.67522.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78706.log.gz 2026-02-22T23:21:00.498 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66408.log 2026-02-22T23:21:00.498 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.60925.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62630.log 2026-02-22T23:21:00.498 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.72557.log.gz 2026-02-22T23:21:00.498 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60925.log.gz 2026-02-22T23:21:00.498 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.66408.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57766.log 2026-02-22T23:21:00.498 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.67522.log.gz 2026-02-22T23:21:00.498 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66408.log.gz 2026-02-22T23:21:00.499 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.62630.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72150.log 2026-02-22T23:21:00.499 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.62630.log.gz 2026-02-22T23:21:00.499 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57766.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59878.log 2026-02-22T23:21:00.499 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.72150.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.57766.log.gz 2026-02-22T23:21:00.499 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66130.log 2026-02-22T23:21:00.499 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.72150.log.gz 2026-02-22T23:21:00.499 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.59878.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48295.log 2026-02-22T23:21:00.499 INFO:teuthology.orchestra.run.trial082.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.59878.log.gz 2026-02-22T23:21:00.500 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78689.log 2026-02-22T23:21:00.500 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.66130.log.gz 2026-02-22T23:21:00.500 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.48295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70922.log 2026-02-22T23:21:00.500 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48295.log.gz 2026-02-22T23:21:00.500 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.78689.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51974.log 2026-02-22T23:21:00.500 INFO:teuthology.orchestra.run.trial082.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.78689.log.gz 2026-02-22T23:21:00.500 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.70922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87873.log 2026-02-22T23:21:00.501 INFO:teuthology.orchestra.run.trial082.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.mirror.0.85432.log.gz 2026-02-22T23:21:00.501 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51974.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82005.log 2026-02-22T23:21:00.501 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.87873.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.51974.log.gz 2026-02-22T23:21:00.501 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.87873.log.gz 2026-02-22T23:21:00.501 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67075.log 2026-02-22T23:21:00.501 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70922.log.gz 2026-02-22T23:21:00.501 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.82005.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81407.log 2026-02-22T23:21:00.501 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.82005.log.gz 2026-02-22T23:21:00.501 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.67075.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78067.log 2026-02-22T23:21:00.502 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.67075.log.gz 2026-02-22T23:21:00.502 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.81407.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74982.log 2026-02-22T23:21:00.502 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.81407.log.gz 2026-02-22T23:21:00.502 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78067.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66785.log 2026-02-22T23:21:00.502 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78067.log.gz 2026-02-22T23:21:00.502 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74982.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74542.log 2026-02-22T23:21:00.502 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74982.log.gz 2026-02-22T23:21:00.503 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.66785.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49603.log 2026-02-22T23:21:00.503 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66785.log.gz 2026-02-22T23:21:00.503 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.74542.log: --verbose -- /var/log/ceph/cluster2-client.admin.71626.log 2026-02-22T23:21:00.503 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49603.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74542.log.gz 2026-02-22T23:21:00.503 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49140.log 2026-02-22T23:21:00.503 INFO:teuthology.orchestra.run.trial082.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49603.log.gz 2026-02-22T23:21:00.503 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.71626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83666.log 2026-02-22T23:21:00.504 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.49140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60829.log 2026-02-22T23:21:00.504 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.71626.log.gz 2026-02-22T23:21:00.504 INFO:teuthology.orchestra.run.trial082.stderr: 80.4% -- replaced with /var/log/ceph/cluster2-client.admin.49140.log.gz 2026-02-22T23:21:00.504 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83666.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75616.log 2026-02-22T23:21:00.504 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.60829.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75900.log 2026-02-22T23:21:00.504 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.60829.log.gz 2026-02-22T23:21:00.505 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61927.log 2026-02-22T23:21:00.505 INFO:teuthology.orchestra.run.trial082.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.75616.log.gz 2026-02-22T23:21:00.505 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75900.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.83666.log.gz 2026-02-22T23:21:00.505 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70842.log 2026-02-22T23:21:00.505 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75900.log.gz 2026-02-22T23:21:00.505 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.82903.log 2026-02-22T23:21:00.505 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80014.log 2026-02-22T23:21:00.505 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.61927.log.gz 2026-02-22T23:21:00.506 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.82903.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.70842.log.gz 2026-02-22T23:21:00.506 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73095.log 2026-02-22T23:21:00.506 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80014.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78019.log 2026-02-22T23:21:00.506 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.80014.log.gz 2026-02-22T23:21:00.506 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73095.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77542.log 2026-02-22T23:21:00.506 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.78019.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65048.log 2026-02-22T23:21:00.506 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78019.log.gz 2026-02-22T23:21:00.507 INFO:teuthology.orchestra.run.trial082.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.82903.log.gz 2026-02-22T23:21:00.507 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81957.log/var/log/ceph/cluster1-client.admin.77542.log: 89.7% 2026-02-22T23:21:00.507 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.73095.log.gz 2026-02-22T23:21:00.507 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61759.log 2026-02-22T23:21:00.507 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.77542.log.gz 2026-02-22T23:21:00.507 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.65048.log.gz 2026-02-22T23:21:00.507 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81957.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85901.log 2026-02-22T23:21:00.507 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.81957.log.gz 2026-02-22T23:21:00.507 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.77120.log 2026-02-22T23:21:00.507 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.61759.log.gz 2026-02-22T23:21:00.508 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.85901.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57668.log 2026-02-22T23:21:00.508 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.85901.log.gz 2026-02-22T23:21:00.508 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.77120.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75082.log 2026-02-22T23:21:00.508 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57668.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77841.log 2026-02-22T23:21:00.508 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.57668.log.gz 2026-02-22T23:21:00.508 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75082.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88380.log 2026-02-22T23:21:00.509 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.77841.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.75082.log.gz 2026-02-22T23:21:00.509 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60270.log 2026-02-22T23:21:00.509 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88380.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86875.log 2026-02-22T23:21:00.509 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88380.log.gz 2026-02-22T23:21:00.509 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60270.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.84133.log 2026-02-22T23:21:00.509 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.60270.log.gz 2026-02-22T23:21:00.510 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.77841.log.gz 2026-02-22T23:21:00.510 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.86875.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72818.log 2026-02-22T23:21:00.510 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.2.84133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65173.log 2026-02-22T23:21:00.510 INFO:teuthology.orchestra.run.trial082.stderr: 19.0% -- replaced with /var/log/ceph/cluster2-client.mirror.2.84133.log.gz 2026-02-22T23:21:00.510 INFO:teuthology.orchestra.run.trial082.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.3.77120.log.gz 2026-02-22T23:21:00.510 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72818.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68409.log 2026-02-22T23:21:00.510 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65173.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57036.log 2026-02-22T23:21:00.510 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.65173.log.gz 2026-02-22T23:21:00.511 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68409.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78264.log 2026-02-22T23:21:00.511 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57036.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.68409.log.gz 2026-02-22T23:21:00.511 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74603.log 2026-02-22T23:21:00.511 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.57036.log.gz 2026-02-22T23:21:00.511 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78264.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49854.log 2026-02-22T23:21:00.511 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72818.log.gz 2026-02-22T23:21:00.511 INFO:teuthology.orchestra.run.trial082.stderr: 87.6%/var/log/ceph/cluster2-client.mirror.74603.log: -- replaced with /var/log/ceph/cluster1-client.mirror.78264.log.gz 2026-02-22T23:21:00.511 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47952.log 2026-02-22T23:21:00.512 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49854.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59457.log 2026-02-22T23:21:00.512 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74603.log.gz 2026-02-22T23:21:00.512 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47952.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52608.log 2026-02-22T23:21:00.512 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47952.log.gz 2026-02-22T23:21:00.512 INFO:teuthology.orchestra.run.trial082.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49854.log.gz 2026-02-22T23:21:00.512 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.59457.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67048.log 2026-02-22T23:21:00.513 INFO:teuthology.orchestra.run.trial082.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.59457.log.gz/var/log/ceph/cluster1-client.admin.52608.log: 2026-02-22T23:21:00.513 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78142.log 2026-02-22T23:21:00.513 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.67048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55207.log 2026-02-22T23:21:00.513 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.52608.log.gz 2026-02-22T23:21:00.513 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.67048.log.gz 2026-02-22T23:21:00.513 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.78142.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68796.log 2026-02-22T23:21:00.513 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.78142.log.gz 2026-02-22T23:21:00.513 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.55207.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51022.log 2026-02-22T23:21:00.514 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68796.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57009.log 2026-02-22T23:21:00.514 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.55207.log.gz 2026-02-22T23:21:00.514 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.51022.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.68796.log.gz 2026-02-22T23:21:00.514 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61786.log 2026-02-22T23:21:00.514 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.51022.log.gz 2026-02-22T23:21:00.514 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63260.log 2026-02-22T23:21:00.514 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.57009.log.gz 2026-02-22T23:21:00.514 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61786.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72210.log 2026-02-22T23:21:00.515 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63260.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.61786.log.gz 2026-02-22T23:21:00.515 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75661.log 2026-02-22T23:21:00.515 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.63260.log.gz 2026-02-22T23:21:00.515 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72210.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53821.log 2026-02-22T23:21:00.515 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75661.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64290.log 2026-02-22T23:21:00.515 INFO:teuthology.orchestra.run.trial082.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75661.log.gz 2026-02-22T23:21:00.516 INFO:teuthology.orchestra.run.trial060.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-22T23:21:00.516 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.53821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82065.log 2026-02-22T23:21:00.516 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.53821.log.gz 2026-02-22T23:21:00.516 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.72210.log.gz 2026-02-22T23:21:00.516 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64290.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67969.log 2026-02-22T23:21:00.516 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.64290.log.gz 2026-02-22T23:21:00.516 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83489.log 2026-02-22T23:21:00.516 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.82065.log: /var/log/ceph/cluster2-client.mirror.67969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49276.log 2026-02-22T23:21:00.516 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67969.log.gz 2026-02-22T23:21:00.516 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.83489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.80418.log 2026-02-22T23:21:00.517 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83489.log.gz 2026-02-22T23:21:00.517 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49276.log: 58.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49276.log.gz 2026-02-22T23:21:00.517 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67650.log 2026-02-22T23:21:00.517 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82065.log.gz 2026-02-22T23:21:00.517 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.80418.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78915.log 2026-02-22T23:21:00.517 INFO:teuthology.orchestra.run.trial082.stderr: 18.5% -- replaced with /var/log/ceph/cluster1-client.mirror.3.80418.log.gz 2026-02-22T23:21:00.517 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.67650.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78091.log 2026-02-22T23:21:00.517 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.67650.log.gz 2026-02-22T23:21:00.518 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69544.log 2026-02-22T23:21:00.518 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.78915.log.gz 2026-02-22T23:21:00.518 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78091.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52662.log 2026-02-22T23:21:00.518 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78091.log.gz 2026-02-22T23:21:00.518 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.69544.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81489.log 2026-02-22T23:21:00.519 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52662.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.69544.log.gz 2026-02-22T23:21:00.519 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56059.log 2026-02-22T23:21:00.519 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81489.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.52662.log.gz 2026-02-22T23:21:00.519 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86855.log 2026-02-22T23:21:00.519 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.56059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53130.log 2026-02-22T23:21:00.519 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.56059.log.gz 2026-02-22T23:21:00.519 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.86855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60350.log 2026-02-22T23:21:00.519 INFO:teuthology.orchestra.run.trial082.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81489.log.gz 2026-02-22T23:21:00.520 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64616.log 2026-02-22T23:21:00.520 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86855.log.gz 2026-02-22T23:21:00.520 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.60350.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86781.log 2026-02-22T23:21:00.520 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.53130.log.gz 2026-02-22T23:21:00.520 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.64616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76904.log 2026-02-22T23:21:00.520 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.60350.log.gz 2026-02-22T23:21:00.521 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64616.log.gz 2026-02-22T23:21:00.521 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48913.log 2026-02-22T23:21:00.521 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.86781.log: /var/log/ceph/cluster2-client.mirror.76904.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86930.log 2026-02-22T23:21:00.521 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.86781.log.gz 2026-02-22T23:21:00.521 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.48913.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66623.log 2026-02-22T23:21:00.521 INFO:teuthology.orchestra.run.trial082.stderr: 57.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48913.log.gz 2026-02-22T23:21:00.521 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.86930.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49157.log 2026-02-22T23:21:00.522 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66623.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86930.log.gz 2026-02-22T23:21:00.522 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.76946.log 2026-02-22T23:21:00.522 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.66623.log.gz 2026-02-22T23:21:00.522 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.49157.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72470.log 2026-02-22T23:21:00.522 INFO:teuthology.orchestra.run.trial082.stderr: 67.0% -- replaced with /var/log/ceph/cluster2-client.admin.49157.log.gz 2026-02-22T23:21:00.522 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.76946.log: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.76946.log.gz 2026-02-22T23:21:00.522 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47070.log 2026-02-22T23:21:00.522 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72470.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84830.log 2026-02-22T23:21:00.523 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47070.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47070.log.gz 2026-02-22T23:21:00.523 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72294.log 2026-02-22T23:21:00.523 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84830.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73054.log 2026-02-22T23:21:00.523 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72294.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75706.log 2026-02-22T23:21:00.523 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.84830.log.gz 2026-02-22T23:21:00.523 INFO:teuthology.orchestra.run.trial082.stderr: 91.9%/var/log/ceph/cluster1-client.mirror.73054.log: -- replaced with /var/log/ceph/cluster1-client.mirror.72470.log.gz 2026-02-22T23:21:00.523 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88821.log 2026-02-22T23:21:00.524 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.72294.log.gz 2026-02-22T23:21:00.524 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75706.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71841.log 2026-02-22T23:21:00.524 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.73054.log.gz 2026-02-22T23:21:00.524 INFO:teuthology.orchestra.run.trial082.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.75706.log.gz 2026-02-22T23:21:00.524 INFO:teuthology.orchestra.run.trial082.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76904.log.gz 2026-02-22T23:21:00.524 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88821.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88821.log.gz 2026-02-22T23:21:00.524 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83893.log 2026-02-22T23:21:00.524 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71841.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66971.log 2026-02-22T23:21:00.524 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83893.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76215.log 2026-02-22T23:21:00.525 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83893.log.gz 2026-02-22T23:21:00.525 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49097.log 2026-02-22T23:21:00.525 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.66971.log.gz 2026-02-22T23:21:00.525 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61381.log 2026-02-22T23:21:00.525 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.49097.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.49097.log.gz 2026-02-22T23:21:00.525 INFO:teuthology.orchestra.run.trial082.stderr:gzip 88.2% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.76215.log.gz --verbose 2026-02-22T23:21:00.525 INFO:teuthology.orchestra.run.trial082.stderr: -- /var/log/ceph/cluster1-client.mirror.81642.log 2026-02-22T23:21:00.526 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61381.log: gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-22T23:21:00.526 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.61381.log.gz 2026-02-22T23:21:00.526 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81642.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80797.log 2026-02-22T23:21:00.526 INFO:teuthology.orchestra.run.trial082.stderr: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.81642.log.gz 2026-02-22T23:21:00.526 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68211.log 2026-02-22T23:21:00.526 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.80797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64215.log 2026-02-22T23:21:00.527 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68211.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.79285.log 2026-02-22T23:21:00.527 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.64215.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.68211.log.gz 2026-02-22T23:21:00.527 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.80797.log.gz 2026-02-22T23:21:00.527 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58045.log 2026-02-22T23:21:00.527 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64215.log.gz 2026-02-22T23:21:00.527 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.79285.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52055.log 2026-02-22T23:21:00.527 INFO:teuthology.orchestra.run.trial082.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71841.log.gz 2026-02-22T23:21:00.527 INFO:teuthology.orchestra.run.trial082.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.79285.log.gz 2026-02-22T23:21:00.528 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58045.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.80347.log 2026-02-22T23:21:00.528 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.58045.log.gz 2026-02-22T23:21:00.528 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52055.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74562.log 2026-02-22T23:21:00.528 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.80347.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57940.log 2026-02-22T23:21:00.528 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.52055.log.gz 2026-02-22T23:21:00.528 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74562.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70083.log 2026-02-22T23:21:00.529 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.57940.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81157.log 2026-02-22T23:21:00.529 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57940.log.gz 2026-02-22T23:21:00.529 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70083.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65101.log 2026-02-22T23:21:00.529 INFO:teuthology.orchestra.run.trial060.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-22T23:21:00.529 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.70083.log.gz 2026-02-22T23:21:00.529 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81157.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47119.log 2026-02-22T23:21:00.529 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.65101.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81157.log.gz 2026-02-22T23:21:00.529 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77400.log 2026-02-22T23:21:00.530 INFO:teuthology.orchestra.run.trial082.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.80347.log.gz 2026-02-22T23:21:00.530 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73668.log 2026-02-22T23:21:00.530 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47119.log.gz 2026-02-22T23:21:00.530 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65101.log.gz 2026-02-22T23:21:00.530 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.77400.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81131.log 2026-02-22T23:21:00.530 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73668.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.77400.log.gz 2026-02-22T23:21:00.530 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55614.log 2026-02-22T23:21:00.531 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81131.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60677.log 93.7% 2026-02-22T23:21:00.531 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-22T23:21:00.531 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73668.log.gz 2026-02-22T23:21:00.531 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.55614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58269.log 2026-02-22T23:21:00.531 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.55614.log.gz 2026-02-22T23:21:00.531 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81131.log.gz 2026-02-22T23:21:00.531 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60677.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62843.log 2026-02-22T23:21:00.531 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.58269.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.60677.log.gz 2026-02-22T23:21:00.531 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52357.log 2026-02-22T23:21:00.532 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58269.log.gz 2026-02-22T23:21:00.532 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74562.log.gz 2026-02-22T23:21:00.532 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62843.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71082.log 2026-02-22T23:21:00.532 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52357.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.62843.log.gz 2026-02-22T23:21:00.532 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69651.log 2026-02-22T23:21:00.532 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52357.log.gz 2026-02-22T23:21:00.532 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.71082.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83753.log 2026-02-22T23:21:00.532 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.69651.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.71082.log.gz 2026-02-22T23:21:00.532 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72597.log 2026-02-22T23:21:00.533 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83753.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58511.log 2026-02-22T23:21:00.533 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69651.log.gz 2026-02-22T23:21:00.533 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83466.log 2026-02-22T23:21:00.533 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72597.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.83753.log.gz 2026-02-22T23:21:00.533 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58511.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61080.log 2026-02-22T23:21:00.534 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.58511.log.gz 2026-02-22T23:21:00.534 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72597.log.gz 2026-02-22T23:21:00.534 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83466.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52140.log 2026-02-22T23:21:00.534 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.61080.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.87121.log 2026-02-22T23:21:00.534 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52140.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64592.log 2026-02-22T23:21:00.534 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61080.log.gz 2026-02-22T23:21:00.534 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.52140.log.gz 2026-02-22T23:21:00.534 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.87121.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61812.log 2026-02-22T23:21:00.535 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.87121.log.gz 2026-02-22T23:21:00.535 INFO:teuthology.orchestra.run.trial082.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-client.mirror.83466.log.gz 2026-02-22T23:21:00.535 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.64592.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88576.log 2026-02-22T23:21:00.535 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.61812.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64592.log.gz 2026-02-22T23:21:00.535 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53418.log 2026-02-22T23:21:00.535 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88576.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64131.log 2026-02-22T23:21:00.535 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88576.log.gz 2026-02-22T23:21:00.535 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61812.log.gz 2026-02-22T23:21:00.536 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.53418.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53418.log.gz 2026-02-22T23:21:00.536 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85059.log 2026-02-22T23:21:00.536 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70272.log 2026-02-22T23:21:00.536 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64131.log: /var/log/ceph/cluster2-client.mirror.85059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76463.log 2026-02-22T23:21:00.536 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.64131.log.gz 2026-02-22T23:21:00.537 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.70272.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86153.log 2026-02-22T23:21:00.537 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70272.log.gz 2026-02-22T23:21:00.537 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72450.log 2026-02-22T23:21:00.537 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.86153.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72493.log 2026-02-22T23:21:00.537 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.76463.log.gz 2026-02-22T23:21:00.537 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72450.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.86153.log.gz 2026-02-22T23:21:00.537 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73313.log 2026-02-22T23:21:00.537 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72493.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66456.log 2026-02-22T23:21:00.538 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73313.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72493.log.gz 2026-02-22T23:21:00.538 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62140.log 2026-02-22T23:21:00.538 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73313.log.gz 2026-02-22T23:21:00.538 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.66456.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85826.log 2026-02-22T23:21:00.538 INFO:teuthology.orchestra.run.trial082.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72450.log.gz 2026-02-22T23:21:00.538 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62140.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63092.log 2026-02-22T23:21:00.538 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.66456.log.gz 2026-02-22T23:21:00.539 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.62140.log.gz 2026-02-22T23:21:00.539 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.85826.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59484.log 2026-02-22T23:21:00.539 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.85826.log.gz 2026-02-22T23:21:00.539 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.63092.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77424.log 2026-02-22T23:21:00.539 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.59484.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57793.log 2026-02-22T23:21:00.539 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.77424.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.59484.log.gz 2026-02-22T23:21:00.539 INFO:teuthology.orchestra.run.trial082.stderr: 95.8% -- replaced with /var/log/ceph/cluster1-client.mirror.1.85467.log.gz 2026-02-22T23:21:00.539 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.63092.log.gz 2026-02-22T23:21:00.539 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.80416.log 2026-02-22T23:21:00.540 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.77424.log.gz 2026-02-22T23:21:00.540 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.57793.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82153.log 2026-02-22T23:21:00.540 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.57793.log.gz 2026-02-22T23:21:00.540 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.80416.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55409.log 2026-02-22T23:21:00.540 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82153.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78569.log 2026-02-22T23:21:00.540 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.55409.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82682.log 2026-02-22T23:21:00.540 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.82153.log.gz 2026-02-22T23:21:00.541 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.55409.log.gz 2026-02-22T23:21:00.541 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.78569.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65726.log 2026-02-22T23:21:00.541 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.78569.log.gz 2026-02-22T23:21:00.541 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82682.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73847.log 2026-02-22T23:21:00.541 INFO:teuthology.orchestra.run.trial082.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.80416.log.gz 2026-02-22T23:21:00.541 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65726.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56710.log 2026-02-22T23:21:00.541 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.65726.log.gz 2026-02-22T23:21:00.542 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73847.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73918.log 2026-02-22T23:21:00.542 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62362.log 2026-02-22T23:21:00.542 INFO:teuthology.orchestra.run.trial082.stderr: 91.2%/var/log/ceph/cluster2-client.admin.56710.log: -- replaced with /var/log/ceph/cluster1-client.mirror.82682.log.gz 2026-02-22T23:21:00.542 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73918.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60650.log 2026-02-22T23:21:00.542 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.56710.log.gz 2026-02-22T23:21:00.542 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.73918.log.gz 2026-02-22T23:21:00.542 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.62362.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50203.log 2026-02-22T23:21:00.543 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60650.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67780.log 2026-02-22T23:21:00.543 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.62362.log.gz 2026-02-22T23:21:00.543 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.60650.log.gz 2026-02-22T23:21:00.543 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.50203.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50161.log 2026-02-22T23:21:00.543 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67780.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.50203.log.gz 2026-02-22T23:21:00.543 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72861.log 2026-02-22T23:21:00.543 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.67780.log.gz 2026-02-22T23:21:00.543 INFO:teuthology.orchestra.run.trial082.stderr:gzip/var/log/ceph/cluster1-client.mirror.50161.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.86180.log 2026-02-22T23:21:00.544 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76696.log 2026-02-22T23:21:00.544 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72861.log.gz 2026-02-22T23:21:00.544 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.86180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56794.log 2026-02-22T23:21:00.544 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.86180.log.gz 2026-02-22T23:21:00.544 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.76696.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74041.log 2026-02-22T23:21:00.544 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.50161.log.gz 2026-02-22T23:21:00.544 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.76696.log.gz 2026-02-22T23:21:00.545 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.56794.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73894.log 2026-02-22T23:21:00.545 INFO:teuthology.orchestra.run.trial082.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73847.log.gz 2026-02-22T23:21:00.545 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56794.log.gz 2026-02-22T23:21:00.545 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.74041.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61113.log 2026-02-22T23:21:00.545 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73894.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55518.log 2026-02-22T23:21:00.545 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.74041.log.gz 2026-02-22T23:21:00.545 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61113.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73894.log.gz 2026-02-22T23:21:00.545 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82629.log 2026-02-22T23:21:00.546 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61113.log.gz 2026-02-22T23:21:00.546 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.55518.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68436.log 2026-02-22T23:21:00.546 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.82629.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54314.log 2026-02-22T23:21:00.547 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82629.log.gz 2026-02-22T23:21:00.547 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68436.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64667.log 2026-02-22T23:21:00.547 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.54314.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.68436.log.gz 2026-02-22T23:21:00.547 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62765.log 2026-02-22T23:21:00.547 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.54314.log.gz 2026-02-22T23:21:00.547 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.64667.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79722.log 2026-02-22T23:21:00.547 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.62765.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.64667.log.gz 2026-02-22T23:21:00.547 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87697.log 2026-02-22T23:21:00.547 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.62765.log.gz 2026-02-22T23:21:00.547 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.79722.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84952.log 2026-02-22T23:21:00.548 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.79722.log.gz 2026-02-22T23:21:00.548 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.87697.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86909.log 2026-02-22T23:21:00.548 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.84952.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.87697.log.gz 2026-02-22T23:21:00.548 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75331.log 2026-02-22T23:21:00.548 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.84952.log.gz 2026-02-22T23:21:00.548 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.86909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66916.log 2026-02-22T23:21:00.548 INFO:teuthology.orchestra.run.trial082.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55518.log.gz 2026-02-22T23:21:00.548 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75331.log: 3.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75331.log.gz 2026-02-22T23:21:00.549 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70896.log 2026-02-22T23:21:00.549 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.66916.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63131.log 2026-02-22T23:21:00.549 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86909.log.gz 2026-02-22T23:21:00.549 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.70896.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.50572.log 2026-02-22T23:21:00.549 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.66916.log.gz 2026-02-22T23:21:00.549 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.70896.log.gz 2026-02-22T23:21:00.550 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.63131.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49349.log 2026-02-22T23:21:00.550 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63131.log.gz 2026-02-22T23:21:00.550 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.50572.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54955.log 2026-02-22T23:21:00.550 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.49349.log: 56.2% -- replaced with /var/log/ceph/cluster2-client.admin.49349.log.gz 2026-02-22T23:21:00.550 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84727.log 2026-02-22T23:21:00.550 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.54955.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55287.log 2026-02-22T23:21:00.550 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84727.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.54955.log.gz 2026-02-22T23:21:00.550 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81110.log 2026-02-22T23:21:00.551 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84727.log.gz 2026-02-22T23:21:00.551 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.55287.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67478.log 2026-02-22T23:21:00.551 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81110.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69885.log 2026-02-22T23:21:00.551 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.67478.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56516.log 2026-02-22T23:21:00.551 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.69885.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79147.log 2026-02-22T23:21:00.552 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67478.log.gz 2026-02-22T23:21:00.552 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.69885.log.gz 2026-02-22T23:21:00.552 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.56516.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55287.log.gz 2026-02-22T23:21:00.552 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67993.log 2026-02-22T23:21:00.552 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.79147.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.56516.log.gz 2026-02-22T23:21:00.552 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.82905.log 2026-02-22T23:21:00.552 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56842.log 2026-02-22T23:21:00.553 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.67993.log: /var/log/ceph/cluster1-client.mirror.2.82905.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67993.log.gz 2026-02-22T23:21:00.553 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73584.log 2026-02-22T23:21:00.553 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.82905.log.gz 2026-02-22T23:21:00.553 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.56842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51848.log 2026-02-22T23:21:00.553 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.56842.log.gz 2026-02-22T23:21:00.553 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.73584.log: /var/log/ceph/cluster1-client.admin.70110.log 2026-02-22T23:21:00.553 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51848.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73584.log.gz 2026-02-22T23:21:00.553 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64994.log 2026-02-22T23:21:00.554 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51848.log.gz 2026-02-22T23:21:00.554 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70110.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72637.log 2026-02-22T23:21:00.554 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64994.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.70110.log.gz 2026-02-22T23:21:00.554 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82424.log 2026-02-22T23:21:00.554 INFO:teuthology.orchestra.run.trial082.stderr: 94.1%/var/log/ceph/cluster2-client.mirror.72637.log: -- replaced with /var/log/ceph/cluster2-client.mirror.81110.log.gz 2026-02-22T23:21:00.554 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.64994.log.gz 2026-02-22T23:21:00.554 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88868.log 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72637.log.gz 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82424.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79857.log 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88868.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88868.log.gz 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52994.log 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.79857.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66324.log 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79147.log.gz 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52994.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.79857.log.gz 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53468.log 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52994.log.gz 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.82424.log.gz 2026-02-22T23:21:00.555 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.66324.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83700.log 2026-02-22T23:21:00.556 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.66324.log.gz 2026-02-22T23:21:00.556 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55825.log/var/log/ceph/cluster1-client.mirror.53468.log: 2026-02-22T23:21:00.556 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53468.log.gz 2026-02-22T23:21:00.556 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83700.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86679.log 2026-02-22T23:21:00.556 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.83700.log.gz 2026-02-22T23:21:00.556 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.55825.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67834.log 2026-02-22T23:21:00.556 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.55825.log.gz 2026-02-22T23:21:00.556 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.86679.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73806.log 2026-02-22T23:21:00.556 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86679.log.gz 2026-02-22T23:21:00.556 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.67834.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56768.log 2026-02-22T23:21:00.557 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73806.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.67834.log.gz 2026-02-22T23:21:00.557 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78043.log 2026-02-22T23:21:00.557 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.56768.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81278.log 2026-02-22T23:21:00.557 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78043.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71652.log 2026-02-22T23:21:00.557 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.78043.log.gz 2026-02-22T23:21:00.557 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56768.log.gz 2026-02-22T23:21:00.558 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81278.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74583.log 2026-02-22T23:21:00.558 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.81278.log.gz 2026-02-22T23:21:00.558 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.71652.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64522.log 2026-02-22T23:21:00.558 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74583.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65985.log 2026-02-22T23:21:00.558 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74583.log.gz 2026-02-22T23:21:00.558 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62968.log 2026-02-22T23:21:00.558 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.64522.log: /var/log/ceph/cluster1-client.mirror.65985.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59854.log 2026-02-22T23:21:00.559 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62968.log: 89.8%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.64522.log.gz -5 2026-02-22T23:21:00.559 INFO:teuthology.orchestra.run.trial082.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.0.79240.log 2026-02-22T23:21:00.559 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65985.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.62968.log.gz 2026-02-22T23:21:00.559 INFO:teuthology.orchestra.run.trial082.stderr: 2026-02-22T23:21:00.559 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58398.log 2026-02-22T23:21:00.559 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.59854.log: /var/log/ceph/cluster1-client.mirror.0.79240.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59854.log.gz 2026-02-22T23:21:00.559 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51416.log 2026-02-22T23:21:00.559 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73806.log.gz 2026-02-22T23:21:00.560 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58398.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83979.log 2026-02-22T23:21:00.560 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.58398.log.gz 2026-02-22T23:21:00.560 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51416.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64747.log 2026-02-22T23:21:00.560 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83979.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.51416.log.gz 2026-02-22T23:21:00.560 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53294.log 2026-02-22T23:21:00.560 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.83979.log.gz 2026-02-22T23:21:00.560 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.64747.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72054.log 2026-02-22T23:21:00.561 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53294.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62941.log 2026-02-22T23:21:00.561 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.64747.log.gz 2026-02-22T23:21:00.561 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53294.log.gz 2026-02-22T23:21:00.561 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72054.log: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.71652.log.gz 2026-02-22T23:21:00.561 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87786.log 2026-02-22T23:21:00.561 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.72054.log.gz 2026-02-22T23:21:00.561 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62941.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68334.log 2026-02-22T23:21:00.561 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.62941.log.gz 2026-02-22T23:21:00.561 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.87786.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73786.log 2026-02-22T23:21:00.562 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.68334.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.84682.log 2026-02-22T23:21:00.562 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.68334.log.gz 2026-02-22T23:21:00.562 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73786.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86464.log 2026-02-22T23:21:00.562 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.84682.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64694.log 2026-02-22T23:21:00.562 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.84682.log.gz 2026-02-22T23:21:00.562 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73786.log.gz 2026-02-22T23:21:00.562 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.86464.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63937.log 2026-02-22T23:21:00.563 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.86464.log.gz 2026-02-22T23:21:00.563 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64694.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47756.log 2026-02-22T23:21:00.563 INFO:teuthology.orchestra.run.trial082.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.79240.log.gz 2026-02-22T23:21:00.563 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.64694.log.gz 2026-02-22T23:21:00.563 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.63937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75815.log 2026-02-22T23:21:00.563 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.47756.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.63937.log.gz 2026-02-22T23:21:00.563 INFO:teuthology.orchestra.run.trial082.stderr:gzip 0.0% -5 -- replaced with /var/log/ceph/cluster1-client.admin.47756.log.gz --verbose 2026-02-22T23:21:00.563 INFO:teuthology.orchestra.run.trial082.stderr: -- /var/log/ceph/cluster1-client.admin.63179.log 2026-02-22T23:21:00.563 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75815.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88478.log 2026-02-22T23:21:00.564 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.63179.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84608.log 2026-02-22T23:21:00.564 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75815.log.gz 2026-02-22T23:21:00.564 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88478.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.63179.log.gz 2026-02-22T23:21:00.564 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88478.log.gz 2026-02-22T23:21:00.564 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56011.log 2026-02-22T23:21:00.564 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84608.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75593.log 2026-02-22T23:21:00.564 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.84608.log.gz 2026-02-22T23:21:00.564 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.56011.log: 96.0% -- replaced with /var/log/ceph/cluster1-client.mirror.87786.log.gz 2026-02-22T23:21:00.565 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49037.log 2026-02-22T23:21:00.565 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56011.log.gz 2026-02-22T23:21:00.565 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75593.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53257.log 2026-02-22T23:21:00.565 INFO:teuthology.orchestra.run.trial082.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.75593.log.gz 2026-02-22T23:21:00.565 INFO:teuthology.orchestra.run.trial082.stderr: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.50572.log.gz 2026-02-22T23:21:00.565 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.49037.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85925.log 2026-02-22T23:21:00.565 INFO:teuthology.orchestra.run.trial082.stderr: 62.0% -- replaced with /var/log/ceph/cluster1-client.admin.49037.log.gz 2026-02-22T23:21:00.565 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53257.log: 66.1% -- replaced with /var/log/ceph/cluster2-client.mirror.53257.log.gz 2026-02-22T23:21:00.565 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75469.log 2026-02-22T23:21:00.565 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.85925.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83509.log 2026-02-22T23:21:00.566 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.85925.log.gz 2026-02-22T23:21:00.566 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73204.log 2026-02-22T23:21:00.566 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75469.log: /var/log/ceph/cluster2-client.mirror.83509.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-22T23:21:00.566 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75469.log.gz 2026-02-22T23:21:00.566 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73204.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59564.log 2026-02-22T23:21:00.566 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56193.log 2026-02-22T23:21:00.567 INFO:teuthology.orchestra.run.trial082.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73204.log.gz 2026-02-22T23:21:00.567 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.59564.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.77062.log 2026-02-22T23:21:00.567 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.83509.log.gz 2026-02-22T23:21:00.567 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.56193.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74665.log 2026-02-22T23:21:00.567 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.59564.log.gz 2026-02-22T23:21:00.567 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.77062.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68963.log 2026-02-22T23:21:00.567 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.77062.log.gz 2026-02-22T23:21:00.567 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72756.log 2026-02-22T23:21:00.567 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74665.log.gz 2026-02-22T23:21:00.568 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55964.log 2026-02-22T23:21:00.568 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.68963.log.gz 2026-02-22T23:21:00.568 INFO:teuthology.orchestra.run.trial082.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-client.mirror.56193.log.gz 2026-02-22T23:21:00.568 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72756.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55234.log 2026-02-22T23:21:00.568 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.55964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69262.log 2026-02-22T23:21:00.568 INFO:teuthology.orchestra.run.trial082.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72756.log.gz 2026-02-22T23:21:00.568 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86260.log 2026-02-22T23:21:00.569 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.55234.log: /var/log/ceph/cluster1-client.admin.69262.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50183.log 2026-02-22T23:21:00.569 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.55234.log.gz 2026-02-22T23:21:00.569 INFO:teuthology.orchestra.run.trial082.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.admin.69262.log.gz 2026-02-22T23:21:00.569 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74419.log/var/log/ceph/cluster1-client.mirror.86260.log: 2026-02-22T23:21:00.569 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.50183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43975.log 2026-02-22T23:21:00.569 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50183.log.gz 2026-02-22T23:21:00.570 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.74419.log: /var/log/ceph/cluster1-client.mirror.87766.log 2026-02-22T23:21:00.570 INFO:teuthology.orchestra.run.trial082.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.86875.log.gz 2026-02-22T23:21:00.570 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.74419.log.gz 2026-02-22T23:21:00.570 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63792.log 2026-02-22T23:21:00.570 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.43975.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43975.log.gz 2026-02-22T23:21:00.570 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76574.log 2026-02-22T23:21:00.570 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.87766.log: /var/log/ceph/cluster1-client.mirror.63792.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48753.log 2026-02-22T23:21:00.570 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.87766.log.gz 2026-02-22T23:21:00.571 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76574.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66510.log 2026-02-22T23:21:00.571 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63792.log.gz 2026-02-22T23:21:00.571 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73224.log 2026-02-22T23:21:00.571 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.48753.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48753.log.gz 2026-02-22T23:21:00.571 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66510.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.55964.log.gz 2026-02-22T23:21:00.571 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83329.log 2026-02-22T23:21:00.571 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76574.log.gz 2026-02-22T23:21:00.571 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.66510.log.gz 2026-02-22T23:21:00.571 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73224.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64566.log 2026-02-22T23:21:00.571 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.54060.log 2026-02-22T23:21:00.572 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.64566.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48879.log 87.7% 2026-02-22T23:21:00.572 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.83329.log.gz 2026-02-22T23:21:00.572 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.3.54060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81555.log 2026-02-22T23:21:00.572 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.48879.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53368.log 2026-02-22T23:21:00.572 INFO:teuthology.orchestra.run.trial082.stderr: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48879.log.gz 2026-02-22T23:21:00.572 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.64566.log.gz 2026-02-22T23:21:00.572 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81555.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73224.log.gz 2026-02-22T23:21:00.572 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68489.log 2026-02-22T23:21:00.572 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53368.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52581.log 2026-02-22T23:21:00.573 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.68489.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70954.log 2026-02-22T23:21:00.573 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52581.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60623.log 2026-02-22T23:21:00.573 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.52581.log.gz 2026-02-22T23:21:00.573 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82489.log 2026-02-22T23:21:00.573 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.68489.log.gz 2026-02-22T23:21:00.573 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.70954.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81555.log.gz 2026-02-22T23:21:00.573 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.60623.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49953.log 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53368.log.gz 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.60623.log.gz 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82489.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62166.log 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70954.log.gz/var/log/ceph/cluster2-client.mirror.49953.log: 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84006.log 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82489.log.gz 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.62166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58647.log 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49953.log.gz 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.84006.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52301.log 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.84006.log.gz 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.58647.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62166.log.gz 2026-02-22T23:21:00.574 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51137.log 2026-02-22T23:21:00.575 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58647.log.gz 2026-02-22T23:21:00.575 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52301.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74377.log 2026-02-22T23:21:00.575 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.51137.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.52301.log.gz 2026-02-22T23:21:00.575 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.79422.log 2026-02-22T23:21:00.575 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74377.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51163.log 2026-02-22T23:21:00.575 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.51137.log.gz 2026-02-22T23:21:00.575 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.79422.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83591.log 2026-02-22T23:21:00.576 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.51163.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61252.log 2026-02-22T23:21:00.576 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51163.log.gz 2026-02-22T23:21:00.576 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83591.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71862.log 2026-02-22T23:21:00.576 INFO:teuthology.orchestra.run.trial082.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.admin.83591.log.gz 2026-02-22T23:21:00.576 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.61252.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64017.log 2026-02-22T23:21:00.576 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67861.log 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.64017.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61252.log.gz 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71862.log.gz 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61900.log 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.67861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66537.log 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.3.79422.log.gz 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.64017.log.gz 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61900.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.67861.log.gz 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71239.log 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.61900.log.gz 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66537.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65324.log 2026-02-22T23:21:00.577 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.66537.log.gz 2026-02-22T23:21:00.578 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71239.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58698.log 2026-02-22T23:21:00.578 INFO:teuthology.orchestra.run.trial082.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74377.log.gz 2026-02-22T23:21:00.578 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.65324.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75311.log 2026-02-22T23:21:00.578 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65324.log.gz 2026-02-22T23:21:00.578 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58698.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69306.log 2026-02-22T23:21:00.578 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.58698.log.gz 2026-02-22T23:21:00.578 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75311.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64781.log 2026-02-22T23:21:00.579 INFO:teuthology.orchestra.run.trial082.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.75311.log.gz 2026-02-22T23:21:00.579 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.69306.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73418.log 2026-02-22T23:21:00.579 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.69306.log.gz 2026-02-22T23:21:00.579 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.64781.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63739.log 2026-02-22T23:21:00.579 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.64781.log.gz 2026-02-22T23:21:00.579 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73418.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69909.log 2026-02-22T23:21:00.579 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63739.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57563.log 2026-02-22T23:21:00.580 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.63739.log.gz 2026-02-22T23:21:00.580 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.73418.log.gz 2026-02-22T23:21:00.580 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.69909.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81789.log 2026-02-22T23:21:00.580 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.69909.log.gz 2026-02-22T23:21:00.580 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.57563.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55879.log 2026-02-22T23:21:00.580 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57563.log.gz 2026-02-22T23:21:00.580 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81789.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67572.log 2026-02-22T23:21:00.580 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81789.log.gz 2026-02-22T23:21:00.580 INFO:teuthology.orchestra.run.trial082.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71239.log.gz 2026-02-22T23:21:00.580 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.55879.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65348.log 2026-02-22T23:21:00.581 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.55879.log.gz 2026-02-22T23:21:00.581 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.67572.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65780.log 2026-02-22T23:21:00.581 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67572.log.gz 2026-02-22T23:21:00.581 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.65348.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74849.log 2026-02-22T23:21:00.581 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65348.log.gz 2026-02-22T23:21:00.581 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65780.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88282.log 2026-02-22T23:21:00.581 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.65780.log.gz 2026-02-22T23:21:00.581 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74849.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.50512.log 2026-02-22T23:21:00.582 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88282.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88282.log.gz 2026-02-22T23:21:00.582 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74849.log.gz 2026-02-22T23:21:00.582 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55129.log 2026-02-22T23:21:00.582 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.50512.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73499.log 2026-02-22T23:21:00.582 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.50512.log.gz 2026-02-22T23:21:00.582 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.55129.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71004.log 2026-02-22T23:21:00.582 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55129.log.gz 2026-02-22T23:21:00.583 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73499.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83352.log 2026-02-22T23:21:00.583 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.71004.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59325.log 2026-02-22T23:21:00.583 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73499.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.71004.log.gz 2026-02-22T23:21:00.583 INFO:teuthology.orchestra.run.trial082.stderr: 2026-02-22T23:21:00.583 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83352.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80726.log 2026-02-22T23:21:00.583 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.83352.log.gz 2026-02-22T23:21:00.583 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.59325.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84583.log 2026-02-22T23:21:00.584 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80726.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.59325.log.gz 2026-02-22T23:21:00.584 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68017.log 2026-02-22T23:21:00.584 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.80726.log.gz 2026-02-22T23:21:00.584 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84583.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85006.log 2026-02-22T23:21:00.584 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.84583.log.gz 2026-02-22T23:21:00.584 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.80312.log 2026-02-22T23:21:00.584 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68017.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.3.54060.log.gz 2026-02-22T23:21:00.584 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.85006.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.68017.log.gzgzip 2026-02-22T23:21:00.585 INFO:teuthology.orchestra.run.trial082.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.admin.47387.log 2026-02-22T23:21:00.585 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.85006.log.gz 2026-02-22T23:21:00.585 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.80312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55566.log 2026-02-22T23:21:00.585 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47387.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47387.log.gz 2026-02-22T23:21:00.585 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67943.log 2026-02-22T23:21:00.585 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.55566.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75571.log 2026-02-22T23:21:00.585 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.67943.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55566.log.gz 2026-02-22T23:21:00.586 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50995.log 2026-02-22T23:21:00.586 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75571.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.50407.log 2026-02-22T23:21:00.586 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67943.log.gz 2026-02-22T23:21:00.586 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.50995.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86484.log 2026-02-22T23:21:00.586 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.50995.log.gz 2026-02-22T23:21:00.586 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.50407.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65513.log 2026-02-22T23:21:00.587 INFO:teuthology.orchestra.run.trial082.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.50407.log.gz 2026-02-22T23:21:00.587 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.86484.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58344.log 2026-02-22T23:21:00.587 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75571.log.gz 2026-02-22T23:21:00.587 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65513.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81748.log 2026-02-22T23:21:00.587 INFO:teuthology.orchestra.run.trial082.stderr: 87.6%/var/log/ceph/cluster2-client.admin.58344.log: -- replaced with /var/log/ceph/cluster1-client.admin.65513.log.gz 2026-02-22T23:21:00.587 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75224.log 2026-02-22T23:21:00.587 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81748.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.58344.log.gz 2026-02-22T23:21:00.587 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49123.log 2026-02-22T23:21:00.588 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75224.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69203.log 2026-02-22T23:21:00.588 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81748.log.gz 2026-02-22T23:21:00.588 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.49123.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73942.log 2026-02-22T23:21:00.588 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75224.log.gz 2026-02-22T23:21:00.588 INFO:teuthology.orchestra.run.trial082.stderr: 80.2% -- replaced with /var/log/ceph/cluster2-client.admin.49123.log.gz 2026-02-22T23:21:00.588 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.69203.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51443.log 2026-02-22T23:21:00.588 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.69203.log.gz 2026-02-22T23:21:00.589 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73942.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61194.log 2026-02-22T23:21:00.589 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73942.log.gz 2026-02-22T23:21:00.589 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51443.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71360.log 2026-02-22T23:21:00.589 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61194.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.51443.log.gz 2026-02-22T23:21:00.589 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54763.log 2026-02-22T23:21:00.589 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.61194.log.gz 2026-02-22T23:21:00.589 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.71360.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.53887.log 2026-02-22T23:21:00.590 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.71360.log.gz 2026-02-22T23:21:00.590 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.54763.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57090.log 2026-02-22T23:21:00.590 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.0.53887.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.54763.log.gz 2026-02-22T23:21:00.590 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69176.log 2026-02-22T23:21:00.590 INFO:teuthology.orchestra.run.trial082.stderr: 95.4% -- replaced with /var/log/ceph/cluster1-client.mirror.86484.log.gz 2026-02-22T23:21:00.590 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.57090.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60464.log 2026-02-22T23:21:00.590 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.57090.log.gz 2026-02-22T23:21:00.590 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.69176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49071.log 2026-02-22T23:21:00.591 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.69176.log.gz 2026-02-22T23:21:00.591 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.60464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64469.log 2026-02-22T23:21:00.591 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.60464.log.gz 2026-02-22T23:21:00.591 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.49071.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.49071.log.gz 2026-02-22T23:21:00.591 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65652.log 2026-02-22T23:21:00.591 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64469.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83957.log 2026-02-22T23:21:00.591 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.65652.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.64469.log.gz 2026-02-22T23:21:00.591 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.50445.log 2026-02-22T23:21:00.592 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83957.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53725.log 2026-02-22T23:21:00.592 INFO:teuthology.orchestra.run.trial082.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.admin.83957.log.gz 2026-02-22T23:21:00.592 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.50445.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65652.log.gz 2026-02-22T23:21:00.592 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59383.log 2026-02-22T23:21:00.592 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53725.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52788.log 2026-02-22T23:21:00.592 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.59383.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53725.log.gz 2026-02-22T23:21:00.592 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84372.log 2026-02-22T23:21:00.593 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70555.log 2026-02-22T23:21:00.593 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.52788.log.gz 2026-02-22T23:21:00.593 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.84372.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.59383.log.gz 2026-02-22T23:21:00.593 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84372.log.gz 2026-02-22T23:21:00.593 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66103.log 2026-02-22T23:21:00.593 INFO:teuthology.orchestra.run.trial082.stderr:gzip/var/log/ceph/cluster1-client.mirror.70555.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76131.log 2026-02-22T23:21:00.593 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66103.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84657.log 2026-02-22T23:21:00.594 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.66103.log.gz 2026-02-22T23:21:00.594 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76131.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52028.log 2026-02-22T23:21:00.594 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.70555.log.gz 2026-02-22T23:21:00.594 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76131.log.gz 2026-02-22T23:21:00.594 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.84657.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.84657.log.gz 2026-02-22T23:21:00.594 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60410.log 2026-02-22T23:21:00.594 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52028.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71715.log 2026-02-22T23:21:00.594 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52028.log.gz 2026-02-22T23:21:00.594 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60410.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64808.log 2026-02-22T23:21:00.595 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.60410.log.gz 2026-02-22T23:21:00.595 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.71715.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.87091.log 2026-02-22T23:21:00.595 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.64808.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57614.log 2026-02-22T23:21:00.595 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.64808.log.gz 2026-02-22T23:21:00.595 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.87091.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87828.log 2026-02-22T23:21:00.595 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.57614.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69284.log 2026-02-22T23:21:00.596 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.71715.log.gz 2026-02-22T23:21:00.596 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.87828.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.57614.log.gz 2026-02-22T23:21:00.596 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84633.log 2026-02-22T23:21:00.596 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.69284.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58962.log 2026-02-22T23:21:00.596 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.69284.log.gz 2026-02-22T23:21:00.596 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87744.log 2026-02-22T23:21:00.596 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.84633.log.gz 2026-02-22T23:21:00.597 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.58962.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51596.log 2026-02-22T23:21:00.597 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.87744.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.80360.log 2026-02-22T23:21:00.597 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58962.log.gz 2026-02-22T23:21:00.597 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51596.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85059.log.gzgzip 2026-02-22T23:21:00.597 INFO:teuthology.orchestra.run.trial082.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.admin.71465.log 2026-02-22T23:21:00.597 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.51596.log.gz 2026-02-22T23:21:00.598 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.80360.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63964.log 2026-02-22T23:21:00.598 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.80360.log.gz 2026-02-22T23:21:00.598 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.71465.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55035.log 2026-02-22T23:21:00.598 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.71465.log.gz 2026-02-22T23:21:00.598 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.87119.log 2026-02-22T23:21:00.598 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.55035.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.63964.log.gz 2026-02-22T23:21:00.598 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54287.log 2026-02-22T23:21:00.598 INFO:teuthology.orchestra.run.trial082.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.87091.log.gz 2026-02-22T23:21:00.599 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.87119.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79210.log 2026-02-22T23:21:00.599 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55035.log.gz 2026-02-22T23:21:00.599 INFO:teuthology.orchestra.run.trial082.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87828.log.gz 2026-02-22T23:21:00.599 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.54287.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78285.log 2026-02-22T23:21:00.599 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.54287.log.gz 2026-02-22T23:21:00.599 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.79210.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60137.log 2026-02-22T23:21:00.599 INFO:teuthology.orchestra.run.trial082.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.79210.log.gz 2026-02-22T23:21:00.599 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78285.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75448.log 2026-02-22T23:21:00.600 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.78285.log.gz 2026-02-22T23:21:00.600 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.60137.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67285.log 2026-02-22T23:21:00.600 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75448.log: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87744.log.gz 2026-02-22T23:21:00.600 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56415.log 2026-02-22T23:21:00.600 INFO:teuthology.orchestra.run.trial082.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.1.87119.log.gz 2026-02-22T23:21:00.600 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60137.log.gz 2026-02-22T23:21:00.600 INFO:teuthology.orchestra.run.trial082.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75448.log.gz 2026-02-22T23:21:00.600 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67285.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75245.log 2026-02-22T23:21:00.600 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.67285.log.gz 2026-02-22T23:21:00.600 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.56415.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66350.log 2026-02-22T23:21:00.601 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75245.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55180.log 2026-02-22T23:21:00.601 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75245.log.gz 2026-02-22T23:21:00.601 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.66350.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56415.log.gz 2026-02-22T23:21:00.601 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82545.log 2026-02-22T23:21:00.601 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49834.log 2026-02-22T23:21:00.601 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.55180.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66350.log.gz 2026-02-22T23:21:00.602 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82545.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.55180.log.gz 2026-02-22T23:21:00.602 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-22T23:21:00.602 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82545.log.gz 2026-02-22T23:21:00.602 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49834.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49834.log.gz 2026-02-22T23:21:00.602 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.85469.log 2026-02-22T23:21:00.602 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51749.log 2026-02-22T23:21:00.603 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.85469.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.85469.log.gz 2026-02-22T23:21:00.603 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57964.log 2026-02-22T23:21:00.603 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51749.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61560.log 2026-02-22T23:21:00.603 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57964.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.51749.log.gz 2026-02-22T23:21:00.603 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59619.log 2026-02-22T23:21:00.603 INFO:teuthology.orchestra.run.trial082.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.57964.log.gz 2026-02-22T23:21:00.603 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61560.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85853.log 2026-02-22T23:21:00.603 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.61560.log.gz/var/log/ceph/cluster1-client.admin.59619.log: 2026-02-22T23:21:00.604 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54601.log 2026-02-22T23:21:00.604 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.59619.log.gz 2026-02-22T23:21:00.604 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.85853.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50917.log 2026-02-22T23:21:00.604 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.85853.log.gz 2026-02-22T23:21:00.604 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.54601.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70529.log 2026-02-22T23:21:00.604 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.54601.log.gz 2026-02-22T23:21:00.604 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.50917.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60324.log 2026-02-22T23:21:00.605 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50917.log.gz 2026-02-22T23:21:00.605 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.70529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81933.log 2026-02-22T23:21:00.605 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.60324.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.70529.log.gz 2026-02-22T23:21:00.605 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56543.log 2026-02-22T23:21:00.605 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.60324.log.gz 2026-02-22T23:21:00.605 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81933.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74897.log 2026-02-22T23:21:00.605 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.81933.log.gz 2026-02-22T23:21:00.605 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.56543.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64344.log 2026-02-22T23:21:00.606 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74897.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.56543.log.gz 2026-02-22T23:21:00.606 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53568.log 2026-02-22T23:21:00.606 INFO:teuthology.orchestra.run.trial082.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74897.log.gz 2026-02-22T23:21:00.606 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.64344.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87849.log 2026-02-22T23:21:00.606 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.64344.log.gz 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53568.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53568.log.gz 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61705.log 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.87849.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.87849.log.gz 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63509.log 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61705.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78939.log 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.61705.log.gz 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.63509.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72841.log 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.63509.log.gz 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.78939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70350.log 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.78939.log.gz 2026-02-22T23:21:00.607 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72841.log: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86260.log.gz 2026-02-22T23:21:00.608 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82108.log 2026-02-22T23:21:00.608 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72841.log.gz 2026-02-22T23:21:00.608 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70350.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70626.log 2026-02-22T23:21:00.608 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.82108.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74710.log 2026-02-22T23:21:00.608 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.70350.log.gz 2026-02-22T23:21:00.608 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.70626.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82108.log.gz 2026-02-22T23:21:00.608 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53543.log 2026-02-22T23:21:00.608 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70626.log.gz 2026-02-22T23:21:00.609 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74710.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68583.log 2026-02-22T23:21:00.609 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74710.log.gz 2026-02-22T23:21:00.609 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53543.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86834.log 2026-02-22T23:21:00.609 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53543.log.gz 2026-02-22T23:21:00.609 INFO:teuthology.orchestra.run.trial082.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.80312.log.gz 2026-02-22T23:21:00.609 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68583.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71545.log 2026-02-22T23:21:00.609 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.68583.log.gz 2026-02-22T23:21:00.609 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.86834.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56113.log 2026-02-22T23:21:00.610 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.71545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53213.log 2026-02-22T23:21:00.610 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.56113.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71801.log 2026-02-22T23:21:00.610 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.71545.log.gz 2026-02-22T23:21:00.610 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.53213.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.56113.log.gz 2026-02-22T23:21:00.610 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.87256.log 2026-02-22T23:21:00.610 INFO:teuthology.orchestra.run.trial082.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53213.log.gz 2026-02-22T23:21:00.610 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71801.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76298.log 2026-02-22T23:21:00.611 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.87256.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77740.log 2026-02-22T23:21:00.611 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71801.log.gz 2026-02-22T23:21:00.611 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.76298.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78409.log 2026-02-22T23:21:00.611 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.77740.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76298.log.gz 2026-02-22T23:21:00.611 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65200.log 2026-02-22T23:21:00.612 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.78409.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.77740.log.gz 2026-02-22T23:21:00.612 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83866.log 2026-02-22T23:21:00.612 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83413.log 2026-02-22T23:21:00.612 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65200.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.78409.log.gz 2026-02-22T23:21:00.612 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72230.log 2026-02-22T23:21:00.612 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.65200.log.gz 2026-02-22T23:21:00.612 INFO:teuthology.orchestra.run.trial082.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.3.87256.log.gz 2026-02-22T23:21:00.612 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.83866.log.gz 2026-02-22T23:21:00.612 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83413.log: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86834.log.gz 2026-02-22T23:21:00.613 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55590.log 2026-02-22T23:21:00.613 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.83413.log.gz 2026-02-22T23:21:00.613 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72230.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48246.log 2026-02-22T23:21:00.613 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.55590.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76624.log 2026-02-22T23:21:00.613 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55590.log.gz 2026-02-22T23:21:00.613 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.48246.log: gzip 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48246.log.gz -5 2026-02-22T23:21:00.613 INFO:teuthology.orchestra.run.trial082.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.68637.log 2026-02-22T23:21:00.613 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.72230.log.gz 2026-02-22T23:21:00.613 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.76624.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65905.log 2026-02-22T23:21:00.614 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.76624.log.gz 2026-02-22T23:21:00.614 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68637.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66079.log 2026-02-22T23:21:00.614 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.68637.log.gz 2026-02-22T23:21:00.614 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65905.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60855.log 2026-02-22T23:21:00.614 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.65905.log.gz 2026-02-22T23:21:00.614 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.66079.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53518.log 2026-02-22T23:21:00.614 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.60855.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66079.log.gz 2026-02-22T23:21:00.614 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84747.log 2026-02-22T23:21:00.615 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53518.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53518.log.gz 2026-02-22T23:21:00.615 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53589.log 2026-02-22T23:21:00.615 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60855.log.gz 2026-02-22T23:21:00.615 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84747.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53393.log 2026-02-22T23:21:00.615 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84747.log.gz 2026-02-22T23:21:00.615 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53589.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53589.log.gzgzip 2026-02-22T23:21:00.615 INFO:teuthology.orchestra.run.trial082.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70222.log 2026-02-22T23:21:00.615 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.53393.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53393.log.gz 2026-02-22T23:21:00.615 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77622.log 2026-02-22T23:21:00.616 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.70222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73651.log 2026-02-22T23:21:00.616 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.77622.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61434.log 2026-02-22T23:21:00.616 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70222.log.gz 2026-02-22T23:21:00.616 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73651.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79911.log 2026-02-22T23:21:00.616 INFO:teuthology.orchestra.run.trial082.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73651.log.gz 2026-02-22T23:21:00.617 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.61434.log: gzip 91.8% -5 -- replaced with /var/log/ceph/cluster1-client.mirror.77622.log.gz --verbose 2026-02-22T23:21:00.617 INFO:teuthology.orchestra.run.trial082.stderr: -- /var/log/ceph/cluster2-client.mirror.75795.log 2026-02-22T23:21:00.617 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.79911.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68463.log 2026-02-22T23:21:00.617 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.79911.log.gz 2026-02-22T23:21:00.617 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.61434.log.gz 2026-02-22T23:21:00.617 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54341.log 2026-02-22T23:21:00.617 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68463.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75795.log.gz 2026-02-22T23:21:00.617 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73520.log 2026-02-22T23:21:00.618 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.68463.log.gz 2026-02-22T23:21:00.618 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.54341.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65702.log 2026-02-22T23:21:00.618 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.54341.log.gz 2026-02-22T23:21:00.618 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73520.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58211.log 2026-02-22T23:21:00.618 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73520.log.gz 2026-02-22T23:21:00.618 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.65702.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76256.log 2026-02-22T23:21:00.618 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.58211.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.65702.log.gz 2026-02-22T23:21:00.618 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86126.log 2026-02-22T23:21:00.619 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76256.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71902.log 2026-02-22T23:21:00.619 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58211.log.gz 2026-02-22T23:21:00.619 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.86126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.82851.log 2026-02-22T23:21:00.619 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.76256.log.gz 2026-02-22T23:21:00.619 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.86126.log.gz 2026-02-22T23:21:00.619 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71902.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75837.log 2026-02-22T23:21:00.619 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.82851.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81326.log 2026-02-22T23:21:00.619 INFO:teuthology.orchestra.run.trial082.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.82851.log.gz 2026-02-22T23:21:00.620 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71902.log.gz 2026-02-22T23:21:00.620 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.75837.log: /var/log/ceph/cluster2-client.mirror.78843.log 2026-02-22T23:21:00.620 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.81326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74316.log 2026-02-22T23:21:00.620 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75837.log.gz 2026-02-22T23:21:00.620 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.81326.log.gz/var/log/ceph/cluster2-client.mirror.78843.log: 2026-02-22T23:21:00.620 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87061.log 2026-02-22T23:21:00.620 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78843.log.gz 2026-02-22T23:21:00.620 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86553.log 2026-02-22T23:21:00.621 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.87061.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71109.log 2026-02-22T23:21:00.621 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53191.log 2026-02-22T23:21:00.621 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.86553.log: /var/log/ceph/cluster1-client.admin.71109.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.86553.log.gz 2026-02-22T23:21:00.621 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76503.log 2026-02-22T23:21:00.621 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.71109.log.gz 2026-02-22T23:21:00.621 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53191.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83613.log 2026-02-22T23:21:00.622 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76503.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61732.log 2026-02-22T23:21:00.622 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48845.log 2026-02-22T23:21:00.622 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.53191.log.gz 2026-02-22T23:21:00.622 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.83613.log.gz 2026-02-22T23:21:00.622 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61732.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51271.log 2026-02-22T23:21:00.622 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.61732.log.gz 2026-02-22T23:21:00.622 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56219.log 2026-02-22T23:21:00.622 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.48845.log: /var/log/ceph/cluster2-client.mirror.51271.log: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48845.log.gz 2026-02-22T23:21:00.622 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55695.log 2026-02-22T23:21:00.623 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.56219.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80912.log 2026-02-22T23:21:00.623 INFO:teuthology.orchestra.run.trial082.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87061.log.gz 2026-02-22T23:21:00.623 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.55695.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73115.log 2026-02-22T23:21:00.623 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56219.log.gz 2026-02-22T23:21:00.623 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.55695.log.gz 2026-02-22T23:21:00.623 INFO:teuthology.orchestra.run.trial082.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51271.log.gz 2026-02-22T23:21:00.623 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58371.log 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80912.log: /var/log/ceph/cluster2-client.mirror.73115.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58158.log 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.80912.log.gz 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74316.log.gz 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73115.log.gz 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58371.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59173.log 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58158.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.58371.log.gz 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48777.log 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.58158.log.gz 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.59173.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78196.log 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.59173.log.gz 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.48777.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51515.log 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76503.log.gz 2026-02-22T23:21:00.624 INFO:teuthology.orchestra.run.trial082.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.admin.48777.log.gz 2026-02-22T23:21:00.625 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.78196.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85401.log 2026-02-22T23:21:00.625 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51515.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61630.log 2026-02-22T23:21:00.625 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.78196.log.gz 2026-02-22T23:21:00.625 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.51515.log.gz 2026-02-22T23:21:00.625 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.85401.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65932.log 2026-02-22T23:21:00.625 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.61630.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.85401.log.gz 2026-02-22T23:21:00.625 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50375.log 2026-02-22T23:21:00.625 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65932.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57315.log 2026-02-22T23:21:00.626 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.65932.log.gz 2026-02-22T23:21:00.626 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.50375.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73182.log 2026-02-22T23:21:00.626 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.61630.log.gz 2026-02-22T23:21:00.626 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.57315.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71943.log 2026-02-22T23:21:00.626 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73182.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51291.log 2026-02-22T23:21:00.626 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.57315.log.gz 2026-02-22T23:21:00.626 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73182.log.gz 2026-02-22T23:21:00.626 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71943.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78645.log 2026-02-22T23:21:00.626 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.51291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69963.log 2026-02-22T23:21:00.627 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.71943.log.gz 2026-02-22T23:21:00.627 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78645.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59959.log 2026-02-22T23:21:00.627 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.69963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70403.log 2026-02-22T23:21:00.627 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.78645.log.gz 2026-02-22T23:21:00.627 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.59959.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.69963.log.gz 2026-02-22T23:21:00.627 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82218.log 2026-02-22T23:21:00.627 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.70403.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.59959.log.gz 2026-02-22T23:21:00.627 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52930.log 2026-02-22T23:21:00.627 INFO:teuthology.orchestra.run.trial082.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51291.log.gz 2026-02-22T23:21:00.627 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82218.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47520.log 2026-02-22T23:21:00.628 INFO:teuthology.orchestra.run.trial082.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82218.log.gz 2026-02-22T23:21:00.628 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70403.log.gz 2026-02-22T23:21:00.628 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.52930.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69122.log 2026-02-22T23:21:00.628 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47520.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62059.log 2026-02-22T23:21:00.628 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47520.log.gz 2026-02-22T23:21:00.628 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52930.log.gz 2026-02-22T23:21:00.628 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.69122.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57846.log 2026-02-22T23:21:00.628 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.62059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78993.log 2026-02-22T23:21:00.628 INFO:teuthology.orchestra.run.trial082.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.69122.log.gz 2026-02-22T23:21:00.628 INFO:teuthology.orchestra.run.trial082.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.50375.log.gz 2026-02-22T23:21:00.629 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.57846.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.62059.log.gz 2026-02-22T23:21:00.629 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47021.log 2026-02-22T23:21:00.629 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.78993.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82264.log 2026-02-22T23:21:00.629 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47021.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.78993.log.gz 2026-02-22T23:21:00.629 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57846.log.gz 2026-02-22T23:21:00.629 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47341.log 2026-02-22T23:21:00.629 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47021.log.gz 2026-02-22T23:21:00.629 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82264.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.76956.log 2026-02-22T23:21:00.629 INFO:teuthology.orchestra.run.trial082.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.82264.log.gz 2026-02-22T23:21:00.629 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47341.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80951.log 2026-02-22T23:21:00.630 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47341.log.gz 2026-02-22T23:21:00.630 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.76956.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.76998.log 2026-02-22T23:21:00.630 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.76956.log.gz 2026-02-22T23:21:00.630 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.80951.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.82860.log 2026-02-22T23:21:00.630 INFO:teuthology.orchestra.run.trial082.stderr: 79.2% -- replaced with /var/log/ceph/cluster2-client.mirror.80951.log.gz 2026-02-22T23:21:00.630 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.76998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74357.log 2026-02-22T23:21:00.630 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.82860.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60111.log 2026-02-22T23:21:00.630 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74357.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54895.log 2026-02-22T23:21:00.631 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.60111.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-22T23:21:00.631 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74357.log.gz 2026-02-22T23:21:00.631 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.60111.log.gz 2026-02-22T23:21:00.631 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51211.log 2026-02-22T23:21:00.631 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.54895.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.76999.log 2026-02-22T23:21:00.631 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-mon.a.log: /var/log/ceph/cluster1-client.admin.51211.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56086.log 2026-02-22T23:21:00.631 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.76999.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.51211.log.gz 2026-02-22T23:21:00.632 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76925.log 2026-02-22T23:21:00.632 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.76999.log.gz 2026-02-22T23:21:00.632 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.56086.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.50397.log 2026-02-22T23:21:00.632 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54895.log.gz 2026-02-22T23:21:00.632 INFO:teuthology.orchestra.run.trial082.stderr: 87.4%/var/log/ceph/cluster2-client.mirror.76925.log: -- replaced with /var/log/ceph/cluster1-client.admin.56086.log.gz 2026-02-22T23:21:00.632 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55267.log 2026-02-22T23:21:00.632 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.50397.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.50397.log.gz 2026-02-22T23:21:00.632 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74808.log 2026-02-22T23:21:00.632 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.55267.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58882.log 2026-02-22T23:21:00.633 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76925.log.gz 2026-02-22T23:21:00.633 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74808.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55436.log 2026-02-22T23:21:00.633 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58882.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54982.log 2026-02-22T23:21:00.633 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.58882.log.gz 2026-02-22T23:21:00.633 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.55436.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71182.log 2026-02-22T23:21:00.633 INFO:teuthology.orchestra.run.trial082.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55267.log.gz 2026-02-22T23:21:00.633 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.54982.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.55436.log.gz 2026-02-22T23:21:00.634 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63533.log 2026-02-22T23:21:00.634 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71182.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.54982.log.gz 2026-02-22T23:21:00.634 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57694.log 2026-02-22T23:21:00.634 INFO:teuthology.orchestra.run.trial082.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71182.log.gz 2026-02-22T23:21:00.634 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63533.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74644.log 2026-02-22T23:21:00.634 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.63533.log.gz 2026-02-22T23:21:00.634 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.57694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60548.log 2026-02-22T23:21:00.634 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74644.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52167.log 2026-02-22T23:21:00.634 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.57694.log.gz 2026-02-22T23:21:00.635 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.60548.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47220.log 2026-02-22T23:21:00.635 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60548.log.gz 2026-02-22T23:21:00.635 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74644.log.gz 2026-02-22T23:21:00.635 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52167.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59409.log 2026-02-22T23:21:00.635 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47220.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65075.log 2026-02-22T23:21:00.635 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47220.log.gz 2026-02-22T23:21:00.635 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.52167.log.gz 2026-02-22T23:21:00.635 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.59409.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84856.log 2026-02-22T23:21:00.635 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59409.log.gz 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65075.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56489.log 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74808.log.gz 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.65075.log.gz 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84856.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86240.log 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.1.50445.log.gz 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84856.log.gz 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.56489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51821.log 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.56489.log.gz/var/log/ceph/cluster2-client.mirror.86240.log: 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59804.log 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86404.log 2026-02-22T23:21:00.636 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.51821.log.gz 2026-02-22T23:21:00.637 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.59804.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49400.log 2026-02-22T23:21:00.637 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.86404.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84533.log 2026-02-22T23:21:00.637 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.86404.log.gz 2026-02-22T23:21:00.637 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.49400.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57481.log 2026-02-22T23:21:00.637 INFO:teuthology.orchestra.run.trial082.stderr: 56.9% -- replaced with /var/log/ceph/cluster2-client.admin.49400.log.gz 2026-02-22T23:21:00.637 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59804.log.gz 2026-02-22T23:21:00.637 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.84533.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76672.log 2026-02-22T23:21:00.637 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84533.log.gz 2026-02-22T23:21:00.638 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.57481.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63712.log 2026-02-22T23:21:00.638 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.76672.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84275.log 2026-02-22T23:21:00.638 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.76672.log.gz 2026-02-22T23:21:00.638 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57481.log.gz 2026-02-22T23:21:00.638 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.63712.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78355.log 2026-02-22T23:21:00.638 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.63712.log.gz 2026-02-22T23:21:00.638 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.84275.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81205.log 2026-02-22T23:21:00.638 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84275.log.gz 2026-02-22T23:21:00.639 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.78355.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50837.log 2026-02-22T23:21:00.639 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81205.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71784.log 2026-02-22T23:21:00.639 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.78355.log.gz 2026-02-22T23:21:00.639 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.81205.log.gz 2026-02-22T23:21:00.639 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.50837.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85248.log 2026-02-22T23:21:00.639 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50837.log.gz 2026-02-22T23:21:00.639 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71784.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49293.log 2026-02-22T23:21:00.639 INFO:teuthology.orchestra.run.trial082.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71784.log.gz 2026-02-22T23:21:00.639 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.85248.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86283.log 2026-02-22T23:21:00.640 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.49293.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83727.log 2026-02-22T23:21:00.640 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.49293.log.gz 2026-02-22T23:21:00.640 INFO:teuthology.orchestra.run.trial082.stderr: 90.2%/var/log/ceph/cluster1-client.mirror.86283.log: -- replaced with /var/log/ceph/cluster2-client.mirror.85248.log.gz 2026-02-22T23:21:00.640 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73709.log 2026-02-22T23:21:00.640 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83727.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55641.log 2026-02-22T23:21:00.640 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86283.log.gz 2026-02-22T23:21:00.640 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73709.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.83727.log.gz 2026-02-22T23:21:00.640 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79189.log 2026-02-22T23:21:00.641 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73709.log.gz 2026-02-22T23:21:00.641 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.55641.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.76998.log.gz 2026-02-22T23:21:00.641 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52221.log 2026-02-22T23:21:00.641 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.55641.log.gz 2026-02-22T23:21:00.641 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.79189.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78243.log 2026-02-22T23:21:00.641 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.79189.log.gz 2026-02-22T23:21:00.641 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52221.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57587.log 2026-02-22T23:21:00.641 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.78243.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.52221.log.gz 2026-02-22T23:21:00.641 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77602.log 2026-02-22T23:21:00.642 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.57587.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60437.log 2026-02-22T23:21:00.642 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.77602.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85974.log 2026-02-22T23:21:00.642 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78243.log.gz 2026-02-22T23:21:00.642 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.57587.log.gz 2026-02-22T23:21:00.642 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.60437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74829.log 2026-02-22T23:21:00.642 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.60437.log.gz 2026-02-22T23:21:00.642 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.85974.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69496.log 2026-02-22T23:21:00.642 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.85974.log.gz 2026-02-22T23:21:00.643 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74829.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77602.log.gz 2026-02-22T23:21:00.643 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78488.log 2026-02-22T23:21:00.643 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.69496.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.50406.log 2026-02-22T23:21:00.643 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74829.log.gz 2026-02-22T23:21:00.643 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69496.log.gz 2026-02-22T23:21:00.643 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.78488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54062.log 2026-02-22T23:21:00.644 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.50406.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.78488.log.gz 2026-02-22T23:21:00.644 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71757.log 2026-02-22T23:21:00.644 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.54062.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79971.log 2026-02-22T23:21:00.644 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71757.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.87197.log 2026-02-22T23:21:00.644 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.79971.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.54062.log.gz 2026-02-22T23:21:00.644 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87642.log 2026-02-22T23:21:00.644 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.87197.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71821.log 2026-02-22T23:21:00.644 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.87197.log.gz 2026-02-22T23:21:00.644 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.87642.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75879.log 2026-02-22T23:21:00.645 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.87642.log.gz 2026-02-22T23:21:00.645 INFO:teuthology.orchestra.run.trial082.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79971.log.gz 2026-02-22T23:21:00.645 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71757.log.gz 2026-02-22T23:21:00.645 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71821.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.77060.log 2026-02-22T23:21:00.645 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71821.log.gz 2026-02-22T23:21:00.645 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75879.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62603.log 2026-02-22T23:21:00.645 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.77060.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86070.log 2026-02-22T23:21:00.645 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75879.log.gz 2026-02-22T23:21:00.646 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62603.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73398.log 2026-02-22T23:21:00.646 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.62603.log.gz 2026-02-22T23:21:00.646 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.86070.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79630.log 2026-02-22T23:21:00.646 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73398.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55079.log 2026-02-22T23:21:00.646 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86070.log.gz 2026-02-22T23:21:00.646 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.79630.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70702.log 2026-02-22T23:21:00.646 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.79630.log.gz 2026-02-22T23:21:00.647 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.55079.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.85433.log 2026-02-22T23:21:00.647 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70702.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65426.log 2026-02-22T23:21:00.647 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.85433.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.70702.log.gz 2026-02-22T23:21:00.647 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87921.log 2026-02-22T23:21:00.647 INFO:teuthology.orchestra.run.trial082.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.85433.log.gz 2026-02-22T23:21:00.647 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55079.log.gz 2026-02-22T23:21:00.647 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65426.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85622.log 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73398.log.gz 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.65426.log.gz 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.87921.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.2.77060.log.gz 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69625.log 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.87921.log.gz 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.85622.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74442.log 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.69625.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.85622.log.gz 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57428.log 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.69625.log.gz 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77569.log 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.57428.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71384.log 2026-02-22T23:21:00.648 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.74442.log.gz 2026-02-22T23:21:00.649 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.57428.log.gz 2026-02-22T23:21:00.649 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.77569.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60899.log 2026-02-22T23:21:00.649 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.71384.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49366.log 2026-02-22T23:21:00.649 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.77569.log.gz 2026-02-22T23:21:00.649 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.60899.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.71384.log.gz 2026-02-22T23:21:00.649 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71310.log 2026-02-22T23:21:00.649 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49366.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55105.log 2026-02-22T23:21:00.649 INFO:teuthology.orchestra.run.trial082.stderr: 55.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49366.log.gz 2026-02-22T23:21:00.650 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60899.log.gz 2026-02-22T23:21:00.650 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71310.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59695.log 2026-02-22T23:21:00.650 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.55105.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75203.log 2026-02-22T23:21:00.650 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55105.log.gz 2026-02-22T23:21:00.650 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71310.log.gz 2026-02-22T23:21:00.650 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.59695.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58538.log 2026-02-22T23:21:00.650 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.59695.log.gz 2026-02-22T23:21:00.650 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75203.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70448.log 2026-02-22T23:21:00.651 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58538.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48896.log 2026-02-22T23:21:00.651 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70448.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.58538.log.gz 2026-02-22T23:21:00.651 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.50510.log 2026-02-22T23:21:00.651 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.70448.log.gz 2026-02-22T23:21:00.651 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.48896.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51542.log 2026-02-22T23:21:00.651 INFO:teuthology.orchestra.run.trial082.stderr: 57.1% -- replaced with /var/log/ceph/cluster1-client.mirror.48896.log.gz 2026-02-22T23:21:00.651 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.50510.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57455.log 2026-02-22T23:21:00.652 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51542.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77902.log 2026-02-22T23:21:00.652 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.57455.log: 90.2%gzip -- replaced with /var/log/ceph/cluster1-client.admin.51542.log.gz 2026-02-22T23:21:00.652 INFO:teuthology.orchestra.run.trial082.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76379.log 2026-02-22T23:21:00.652 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.57455.log.gz 2026-02-22T23:21:00.652 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.77902.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72577.log 2026-02-22T23:21:00.652 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77902.log.gz 2026-02-22T23:21:00.652 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76379.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67371.log 2026-02-22T23:21:00.653 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72577.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76318.log 2026-02-22T23:21:00.653 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67371.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.72577.log.gz 2026-02-22T23:21:00.653 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64721.log 2026-02-22T23:21:00.653 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75203.log.gz 2026-02-22T23:21:00.653 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.67371.log.gz 2026-02-22T23:21:00.653 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76318.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76379.log.gz 2026-02-22T23:21:00.653 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81687.log 2026-02-22T23:21:00.654 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64721.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51362.log 2026-02-22T23:21:00.654 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76318.log.gz 2026-02-22T23:21:00.654 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.64721.log.gz 2026-02-22T23:21:00.654 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81687.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76152.log 2026-02-22T23:21:00.654 INFO:teuthology.orchestra.run.trial082.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81687.log.gz 2026-02-22T23:21:00.654 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51362.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82344.log 2026-02-22T23:21:00.654 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.51362.log.gz 2026-02-22T23:21:00.654 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74074.log 2026-02-22T23:21:00.655 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76152.log.gz 2026-02-22T23:21:00.655 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.82344.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59018.log 2026-02-22T23:21:00.655 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74074.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66994.log 2026-02-22T23:21:00.655 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82344.log.gz 2026-02-22T23:21:00.655 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.59018.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73606.log 2026-02-22T23:21:00.656 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66994.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66761.log 2026-02-22T23:21:00.656 INFO:teuthology.orchestra.run.trial082.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74074.log.gz 2026-02-22T23:21:00.656 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.66994.log.gz 2026-02-22T23:21:00.656 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59018.log.gz 2026-02-22T23:21:00.656 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73606.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62412.log 2026-02-22T23:21:00.656 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73606.log.gz 2026-02-22T23:21:00.656 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.66761.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.82965.log 2026-02-22T23:21:00.656 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66761.log.gz 2026-02-22T23:21:00.656 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.62412.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58909.log 2026-02-22T23:21:00.657 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62412.log.gz 2026-02-22T23:21:00.657 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.82965.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63400.log 2026-02-22T23:21:00.657 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58909.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59511.log 2026-02-22T23:21:00.657 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.58909.log.gz 2026-02-22T23:21:00.657 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63400.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62490.log 2026-02-22T23:21:00.657 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.63400.log.gz 2026-02-22T23:21:00.657 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.59511.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55668.log 2026-02-22T23:21:00.658 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.59511.log.gz 2026-02-22T23:21:00.658 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.62490.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.84131.log 2026-02-22T23:21:00.658 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.62490.log.gz 2026-02-22T23:21:00.658 INFO:teuthology.orchestra.run.trial082.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.82965.log.gz 2026-02-22T23:21:00.658 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.55668.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67183.log 2026-02-22T23:21:00.658 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.55668.log.gz 2026-02-22T23:21:00.658 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.1.84131.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68556.log 2026-02-22T23:21:00.658 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.67183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64104.log 2026-02-22T23:21:00.659 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67183.log.gz 2026-02-22T23:21:00.659 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68556.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56345.log 2026-02-22T23:21:00.659 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.68556.log.gz 2026-02-22T23:21:00.659 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64104.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74192.log 2026-02-22T23:21:00.659 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.64104.log.gz 2026-02-22T23:21:00.659 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.56345.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63685.log 2026-02-22T23:21:00.659 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.56345.log.gz 2026-02-22T23:21:00.660 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74192.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81254.log 2026-02-22T23:21:00.660 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.63685.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68071.log 2026-02-22T23:21:00.660 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.63685.log.gz 2026-02-22T23:21:00.660 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70190.log 2026-02-22T23:21:00.660 INFO:teuthology.orchestra.run.trial082.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.74192.log.gz 2026-02-22T23:21:00.660 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81254.log.gz 2026-02-22T23:21:00.660 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68071.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76026.log 2026-02-22T23:21:00.661 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.68071.log.gz 2026-02-22T23:21:00.661 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.70190.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80776.log 2026-02-22T23:21:00.661 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76026.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75045.log 2026-02-22T23:21:00.661 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70190.log.gz 2026-02-22T23:21:00.661 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76026.log.gz 2026-02-22T23:21:00.661 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80776.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.80776.log.gz 2026-02-22T23:21:00.661 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66382.log 2026-02-22T23:21:00.662 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56596.log 2026-02-22T23:21:00.662 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75045.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75045.log.gz 2026-02-22T23:21:00.662 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.66382.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59244.log 2026-02-22T23:21:00.662 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.56596.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69387.log 2026-02-22T23:21:00.662 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.66382.log.gz 2026-02-22T23:21:00.662 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.59244.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61586.log 2026-02-22T23:21:00.663 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.56596.log.gz 2026-02-22T23:21:00.663 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.59244.log.gz 2026-02-22T23:21:00.663 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.69387.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.87093.log 2026-02-22T23:21:00.663 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.69387.log.gz 2026-02-22T23:21:00.663 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.61586.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.44022.log 2026-02-22T23:21:00.663 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.87093.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61408.log 2026-02-22T23:21:00.663 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.87093.log.gz 2026-02-22T23:21:00.664 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.44022.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61586.log.gz 2026-02-22T23:21:00.664 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.44022.log.gz 2026-02-22T23:21:00.664 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61873.log 2026-02-22T23:21:00.664 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61408.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48442.log 2026-02-22T23:21:00.664 INFO:teuthology.orchestra.run.trial082.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.1.84131.log.gz 2026-02-22T23:21:00.664 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61408.log.gz 2026-02-22T23:21:00.664 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61873.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57374.log 2026-02-22T23:21:00.664 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.61873.log.gz 2026-02-22T23:21:00.664 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.48442.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48442.log.gz 2026-02-22T23:21:00.664 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85033.log 2026-02-22T23:21:00.665 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57374.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79991.log 2026-02-22T23:21:00.665 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.57374.log.gz 2026-02-22T23:21:00.665 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.85033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78817.log 2026-02-22T23:21:00.665 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.85033.log.gz 2026-02-22T23:21:00.665 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.79991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68098.log 2026-02-22T23:21:00.666 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.78817.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70137.log 2026-02-22T23:21:00.666 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68098.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64640.log 2026-02-22T23:21:00.666 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78817.log.gz 2026-02-22T23:21:00.666 INFO:teuthology.orchestra.run.trial082.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79991.log.gz 2026-02-22T23:21:00.666 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.68098.log.gz 2026-02-22T23:21:00.666 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.70137.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.53878.log 2026-02-22T23:21:00.666 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.70137.log.gz 2026-02-22T23:21:00.666 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.64640.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56656.log 2026-02-22T23:21:00.667 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.0.53878.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.64640.log.gz 2026-02-22T23:21:00.667 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.53878.log.gz 2026-02-22T23:21:00.667 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66949.log 2026-02-22T23:21:00.667 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.56656.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82196.log 2026-02-22T23:21:00.667 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.56656.log.gz 2026-02-22T23:21:00.667 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50003.log 2026-02-22T23:21:00.667 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.66949.log.gz 2026-02-22T23:21:00.668 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82196.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72924.log 2026-02-22T23:21:00.668 INFO:teuthology.orchestra.run.trial082.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82196.log.gz 2026-02-22T23:21:00.668 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.50003.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62896.log 2026-02-22T23:21:00.668 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50003.log.gz 2026-02-22T23:21:00.668 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72318.log 2026-02-22T23:21:00.668 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.62896.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72924.log.gz 2026-02-22T23:21:00.668 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55462.log 2026-02-22T23:21:00.669 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72318.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67398.log 2026-02-22T23:21:00.669 INFO:teuthology.orchestra.run.trial082.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.2.50510.log.gz 2026-02-22T23:21:00.669 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62896.log.gz 2026-02-22T23:21:00.669 INFO:teuthology.orchestra.run.trial082.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.72318.log.gz 2026-02-22T23:21:00.669 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.55462.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86703.log 2026-02-22T23:21:00.669 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55462.log.gz 2026-02-22T23:21:00.669 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63459.log 2026-02-22T23:21:00.669 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67398.log: /var/log/ceph/cluster1-client.mirror.86703.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86432.log 2026-02-22T23:21:00.670 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.67398.log.gz 2026-02-22T23:21:00.670 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.86703.log.gz 2026-02-22T23:21:00.670 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.63459.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81768.log 2026-02-22T23:21:00.670 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.86432.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67231.log 2026-02-22T23:21:00.670 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.86432.log.gz 2026-02-22T23:21:00.670 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63459.log.gz 2026-02-22T23:21:00.670 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81768.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65678.log 2026-02-22T23:21:00.671 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81768.log.gz 2026-02-22T23:21:00.671 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.67231.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58018.log 2026-02-22T23:21:00.671 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.67231.log.gz 2026-02-22T23:21:00.671 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.65678.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60243.log 2026-02-22T23:21:00.671 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65678.log.gz 2026-02-22T23:21:00.671 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58018.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73291.log 2026-02-22T23:21:00.671 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.58018.log.gz 2026-02-22T23:21:00.671 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60243.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54736.log 2026-02-22T23:21:00.672 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.60243.log.gz 2026-02-22T23:21:00.672 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78169.log 2026-02-22T23:21:00.672 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73291.log.gz 2026-02-22T23:21:00.672 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.54736.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47854.log 2026-02-22T23:21:00.672 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.54736.log.gz 2026-02-22T23:21:00.672 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.78169.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86727.log 2026-02-22T23:21:00.672 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.78169.log.gz 2026-02-22T23:21:00.672 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.47854.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47854.log.gz 2026-02-22T23:21:00.673 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60775.log 2026-02-22T23:21:00.673 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.86727.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52815.log 2026-02-22T23:21:00.673 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.86727.log.gz 2026-02-22T23:21:00.673 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60775.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.49533.log 2026-02-22T23:21:00.673 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.60775.log.gz 2026-02-22T23:21:00.673 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52815.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62292.log 2026-02-22T23:21:00.673 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.49533.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.52815.log.gz 2026-02-22T23:21:00.674 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84558.log 2026-02-22T23:21:00.674 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.62292.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54473.log 2026-02-22T23:21:00.674 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.62292.log.gz 2026-02-22T23:21:00.674 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.84558.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84558.log.gz 2026-02-22T23:21:00.674 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75530.log 2026-02-22T23:21:00.674 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.54473.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60219.log 2026-02-22T23:21:00.674 INFO:teuthology.orchestra.run.trial082.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.49533.log.gz 2026-02-22T23:21:00.675 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75530.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68745.log 2026-02-22T23:21:00.675 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54473.log.gz 2026-02-22T23:21:00.675 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75530.log.gz 2026-02-22T23:21:00.675 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.60219.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71737.log 2026-02-22T23:21:00.675 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60219.log.gz 2026-02-22T23:21:00.675 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.68745.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.44149.log 2026-02-22T23:21:00.675 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.68745.log.gz 2026-02-22T23:21:00.675 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71737.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74768.log 2026-02-22T23:21:00.676 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71737.log.gz 2026-02-22T23:21:00.676 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.44149.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.44149.log.gz 2026-02-22T23:21:00.676 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.84193.log 2026-02-22T23:21:00.676 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74768.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50968.log 2026-02-22T23:21:00.676 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.3.84193.log: 19.0% -- replaced with /var/log/ceph/cluster2-client.mirror.3.84193.log.gz 2026-02-22T23:21:00.676 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81089.log 2026-02-22T23:21:00.676 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74768.log.gz 2026-02-22T23:21:00.676 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.50968.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83817.log 2026-02-22T23:21:00.677 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81089.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.50968.log.gz 2026-02-22T23:21:00.677 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76236.log 2026-02-22T23:21:00.677 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75638.log 2026-02-22T23:21:00.677 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.83817.log.gz 2026-02-22T23:21:00.677 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76236.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69333.log 2026-02-22T23:21:00.677 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76236.log.gz 2026-02-22T23:21:00.678 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75638.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65567.log 2026-02-22T23:21:00.678 INFO:teuthology.orchestra.run.trial082.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75638.log.gz 2026-02-22T23:21:00.678 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.69333.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.86240.log.gz 2026-02-22T23:21:00.678 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47903.log 2026-02-22T23:21:00.678 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.69333.log.gz 2026-02-22T23:21:00.678 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65567.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82602.log 2026-02-22T23:21:00.678 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.65567.log.gz 2026-02-22T23:21:00.678 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47903.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47903.log.gz 2026-02-22T23:21:00.678 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82568.log 2026-02-22T23:21:00.679 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.82602.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74464.log 2026-02-22T23:21:00.679 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.82602.log.gz 2026-02-22T23:21:00.679 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82568.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47805.log 2026-02-22T23:21:00.679 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82568.log.gz 2026-02-22T23:21:00.679 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52865.log 2026-02-22T23:21:00.679 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.74464.log.gz 2026-02-22T23:21:00.679 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.47805.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47805.log.gz 2026-02-22T23:21:00.680 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54843.log 2026-02-22T23:21:00.680 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.52865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52909.log 2026-02-22T23:21:00.680 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52865.log.gz 2026-02-22T23:21:00.680 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81089.log.gz 2026-02-22T23:21:00.680 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.54843.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.49432.log 2026-02-22T23:21:00.680 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.52909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50054.log 2026-02-22T23:21:00.680 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.54843.log.gz 2026-02-22T23:21:00.681 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.49432.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52909.log.gz 2026-02-22T23:21:00.681 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- 19.0% /var/log/ceph/cluster2-client.mirror.59199.log 2026-02-22T23:21:00.681 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.1.49432.log.gz 2026-02-22T23:21:00.681 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.50054.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74688.log 2026-02-22T23:21:00.681 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.50054.log.gz 2026-02-22T23:21:00.681 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.59199.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70756.log 2026-02-22T23:21:00.681 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74688.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63346.log 2026-02-22T23:21:00.681 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59199.log.gz 2026-02-22T23:21:00.681 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.74688.log.gz 2026-02-22T23:21:00.682 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.70756.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67703.log 2026-02-22T23:21:00.682 INFO:teuthology.orchestra.run.trial082.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.53887.log.gz 2026-02-22T23:21:00.682 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.70756.log.gz 2026-02-22T23:21:00.682 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.63346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66890.log 2026-02-22T23:21:00.682 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.63346.log.gz 2026-02-22T23:21:00.682 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.67703.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68909.log 2026-02-22T23:21:00.682 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.66890.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62318.log 2026-02-22T23:21:00.682 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.67703.log.gz 2026-02-22T23:21:00.682 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.66890.log.gz 2026-02-22T23:21:00.683 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59271.log 2026-02-22T23:21:00.683 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.68909.log.gz 2026-02-22T23:21:00.683 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.62318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62265.log 2026-02-22T23:21:00.683 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.59271.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51187.log 2026-02-22T23:21:00.683 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.59271.log.gz 2026-02-22T23:21:00.683 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62318.log.gz 2026-02-22T23:21:00.683 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.62265.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62086.log 2026-02-22T23:21:00.684 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.62265.log.gz 2026-02-22T23:21:00.684 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.51187.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63587.log 2026-02-22T23:21:00.684 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51187.log.gz 2026-02-22T23:21:00.684 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.62086.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88527.log 2026-02-22T23:21:00.684 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.62086.log.gz 2026-02-22T23:21:00.684 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.63587.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75268.log 2026-02-22T23:21:00.684 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.63587.log.gz 2026-02-22T23:21:00.684 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88527.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88527.log.gz 2026-02-22T23:21:00.684 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54547.log 2026-02-22T23:21:00.685 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75268.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53274.log 2026-02-22T23:21:00.685 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75268.log.gz 2026-02-22T23:21:00.685 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.54547.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67101.log 2026-02-22T23:21:00.685 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.54547.log.gz 2026-02-22T23:21:00.685 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69571.log 2026-02-22T23:21:00.685 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.67101.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55906.log 2026-02-22T23:21:00.685 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53274.log.gz 2026-02-22T23:21:00.686 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.69571.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78622.log 2026-02-22T23:21:00.686 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67101.log.gz 2026-02-22T23:21:00.686 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.69571.log.gz 2026-02-22T23:21:00.686 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.55906.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52734.log 2026-02-22T23:21:00.686 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.55906.log.gz 2026-02-22T23:21:00.686 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78622.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75963.log 2026-02-22T23:21:00.687 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82032.log 2026-02-22T23:21:00.687 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.52734.log.gz 2026-02-22T23:21:00.687 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75963.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58564.log 2026-02-22T23:21:00.687 INFO:teuthology.orchestra.run.trial082.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78622.log.gz 2026-02-22T23:21:00.687 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75963.log.gz 2026-02-22T23:21:00.687 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.82032.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58597.log 2026-02-22T23:21:00.687 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82032.log.gz 2026-02-22T23:21:00.687 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.58564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49383.log 2026-02-22T23:21:00.688 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.58597.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76173.log 2026-02-22T23:21:00.688 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.49383.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58564.log.gz 2026-02-22T23:21:00.688 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88772.log 2026-02-22T23:21:00.688 INFO:teuthology.orchestra.run.trial082.stderr: 63.0% -- replaced with /var/log/ceph/cluster2-client.admin.49383.log.gz 2026-02-22T23:21:00.688 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76173.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.58597.log.gz 2026-02-22T23:21:00.688 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53170.log 2026-02-22T23:21:00.688 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76173.log.gz 2026-02-22T23:21:00.688 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88772.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88772.log.gz 2026-02-22T23:21:00.688 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65298.log 2026-02-22T23:21:00.689 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53170.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59433.log 2026-02-22T23:21:00.689 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53170.log.gz 2026-02-22T23:21:00.689 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.65298.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70675.log 2026-02-22T23:21:00.689 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.59433.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68936.log 2026-02-22T23:21:00.689 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59433.log.gz 2026-02-22T23:21:00.689 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70675.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.65298.log.gz 2026-02-22T23:21:00.689 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63022.log 2026-02-22T23:21:00.690 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.70675.log.gz 2026-02-22T23:21:00.690 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68936.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59068.log 2026-02-22T23:21:00.690 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.68936.log.gz 2026-02-22T23:21:00.690 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63022.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.80313.log 2026-02-22T23:21:00.690 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.63022.log.gz 2026-02-22T23:21:00.690 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.59068.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56818.log 2026-02-22T23:21:00.690 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.59068.log.gz 2026-02-22T23:21:00.690 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.80313.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73458.log 2026-02-22T23:21:00.690 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.80313.log.gz 2026-02-22T23:21:00.691 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.56818.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64835.log 2026-02-22T23:21:00.691 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.56818.log.gz 2026-02-22T23:21:00.691 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73458.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60490.log 2026-02-22T23:21:00.691 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64835.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63373.log 2026-02-22T23:21:00.691 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73458.log.gz 2026-02-22T23:21:00.691 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.64835.log.gz 2026-02-22T23:21:00.692 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.60490.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66243.log 2026-02-22T23:21:00.692 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63373.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86573.log 2026-02-22T23:21:00.692 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60490.log.gz 2026-02-22T23:21:00.692 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.63373.log.gz 2026-02-22T23:21:00.692 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66243.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49259.log 2026-02-22T23:21:00.692 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.66243.log.gz 2026-02-22T23:21:00.692 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.86573.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62995.log 2026-02-22T23:21:00.692 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49259.log: 58.0% -- replaced with /var/log/ceph/cluster2-client.mirror.49259.log.gz 2026-02-22T23:21:00.693 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48729.log 2026-02-22T23:21:00.693 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.62995.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70016.log 2026-02-22T23:21:00.693 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.62995.log.gz 2026-02-22T23:21:00.693 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.48729.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48729.log.gz 2026-02-22T23:21:00.693 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70729.log 2026-02-22T23:21:00.694 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.70016.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82830.log 2026-02-22T23:21:00.694 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.70729.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70980.log 2026-02-22T23:21:00.694 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70016.log.gz 2026-02-22T23:21:00.694 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.70729.log.gz 2026-02-22T23:21:00.694 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81707.log 2026-02-22T23:21:00.694 INFO:teuthology.orchestra.run.trial082.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82830.log.gz 2026-02-22T23:21:00.694 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.70980.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67452.log 2026-02-22T23:21:00.694 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81707.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70980.log.gz 2026-02-22T23:21:00.694 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58424.log 2026-02-22T23:21:00.695 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.67452.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66483.log 2026-02-22T23:21:00.695 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.67452.log.gz 2026-02-22T23:21:00.695 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.58424.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84904.log 2026-02-22T23:21:00.695 INFO:teuthology.orchestra.run.trial082.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81707.log.gz 2026-02-22T23:21:00.695 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.86573.log.gz 2026-02-22T23:21:00.695 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.66483.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72904.log 2026-02-22T23:21:00.695 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.58424.log.gz 2026-02-22T23:21:00.695 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.84904.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83252.log 2026-02-22T23:21:00.696 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.66483.log.gz 2026-02-22T23:21:00.696 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84904.log.gz 2026-02-22T23:21:00.696 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72904.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62543.log 2026-02-22T23:21:00.696 INFO:teuthology.orchestra.run.trial082.stderr: 95.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.82860.log.gz 2026-02-22T23:21:00.696 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83252.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59044.log 2026-02-22T23:21:00.696 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72904.log.gz 2026-02-22T23:21:00.696 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.83252.log.gz 2026-02-22T23:21:00.696 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.62543.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86516.log 2026-02-22T23:21:00.697 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.59044.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52194.log 2026-02-22T23:21:00.697 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.62543.log.gz 2026-02-22T23:21:00.697 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59044.log.gz 2026-02-22T23:21:00.697 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.86516.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63910.log 2026-02-22T23:21:00.697 INFO:teuthology.orchestra.run.trial082.stderr: 79.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86516.log.gz 2026-02-22T23:21:00.697 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52194.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75775.log 2026-02-22T23:21:00.697 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.52194.log.gz 2026-02-22T23:21:00.698 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.63910.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78435.log 2026-02-22T23:21:00.698 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.63910.log.gz 2026-02-22T23:21:00.698 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75775.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69861.log 2026-02-22T23:21:00.698 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78435.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68358.log 2026-02-22T23:21:00.698 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75775.log.gz 2026-02-22T23:21:00.698 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.69861.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62715.log 2026-02-22T23:21:00.698 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78435.log.gz 2026-02-22T23:21:00.698 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69861.log.gz/var/log/ceph/cluster2-client.mirror.68358.log: 2026-02-22T23:21:00.698 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65594.log 2026-02-22T23:21:00.698 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68358.log.gz 2026-02-22T23:21:00.699 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.62715.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77882.log 2026-02-22T23:21:00.699 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65594.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62576.log 2026-02-22T23:21:00.699 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.65594.log.gz 2026-02-22T23:21:00.699 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.77882.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82310.log 2026-02-22T23:21:00.699 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62715.log.gz 2026-02-22T23:21:00.699 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62576.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77882.log.gz 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68850.log 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82310.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.62576.log.gz 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75143.log 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82310.log.gz 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68850.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65807.log 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75143.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.68850.log.gz 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86754.log 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65807.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75143.log.gz 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69413.log 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.65807.log.gz 2026-02-22T23:21:00.700 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.86754.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56291.log 2026-02-22T23:21:00.701 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.86754.log.gz 2026-02-22T23:21:00.701 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.69413.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75984.log 2026-02-22T23:21:00.701 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.56291.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84928.log 2026-02-22T23:21:00.701 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69413.log.gz 2026-02-22T23:21:00.701 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.56291.log.gz 2026-02-22T23:21:00.701 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75984.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81597.log 2026-02-22T23:21:00.701 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75984.log.gz/var/log/ceph/cluster1-client.mirror.84928.log: 2026-02-22T23:21:00.702 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49208.log 2026-02-22T23:21:00.702 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84928.log.gz 2026-02-22T23:21:00.702 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81597.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67596.log 2026-02-22T23:21:00.702 INFO:teuthology.orchestra.run.trial082.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81597.log.gz 2026-02-22T23:21:00.702 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49208.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49208.log.gz 2026-02-22T23:21:00.702 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71572.log 2026-02-22T23:21:00.702 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67596.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81302.log 2026-02-22T23:21:00.702 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.67596.log.gz 2026-02-22T23:21:00.703 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.71572.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66735.log 2026-02-22T23:21:00.703 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.71572.log.gz 2026-02-22T23:21:00.703 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81302.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64050.log 2026-02-22T23:21:00.703 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.81302.log.gz 2026-02-22T23:21:00.703 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.66735.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52761.log 2026-02-22T23:21:00.703 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.64050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68044.log 2026-02-22T23:21:00.704 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.64050.log.gz 2026-02-22T23:21:00.704 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52761.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51238.log 2026-02-22T23:21:00.704 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.66735.log.gz 2026-02-22T23:21:00.704 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68044.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.52761.log.gz 2026-02-22T23:21:00.704 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68663.log 2026-02-22T23:21:00.704 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51238.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.68044.log.gz 2026-02-22T23:21:00.704 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65959.log 2026-02-22T23:21:00.704 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.68663.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.51238.log.gz 2026-02-22T23:21:00.705 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55354.log 2026-02-22T23:21:00.705 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65959.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64370.log 2026-02-22T23:21:00.705 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68663.log.gz 2026-02-22T23:21:00.705 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.65959.log.gz 2026-02-22T23:21:00.705 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.55354.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64263.log 2026-02-22T23:21:00.705 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.55354.log.gz 2026-02-22T23:21:00.705 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.64370.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.49474.log 2026-02-22T23:21:00.706 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64263.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72126.log 2026-02-22T23:21:00.706 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.64263.log.gz 2026-02-22T23:21:00.706 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.49474.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64370.log.gz 2026-02-22T23:21:00.706 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67736.log 2026-02-22T23:21:00.706 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49003.log 2026-02-22T23:21:00.706 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72126.log.gz 2026-02-22T23:21:00.706 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67736.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81045.log 2026-02-22T23:21:00.706 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.67736.log.gz 2026-02-22T23:21:00.707 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.49003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81380.log 2026-02-22T23:21:00.707 INFO:teuthology.orchestra.run.trial082.stderr: 56.1% -- replaced with /var/log/ceph/cluster1-client.admin.49003.log.gz 2026-02-22T23:21:00.707 INFO:teuthology.orchestra.run.trial082.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.49474.log.gz 2026-02-22T23:21:00.707 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81045.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69149.log 2026-02-22T23:21:00.707 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.81380.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.81045.log.gz 2026-02-22T23:21:00.707 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66210.log 2026-02-22T23:21:00.707 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.81380.log.gz 2026-02-22T23:21:00.707 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.69149.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74215.log 2026-02-22T23:21:00.707 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.69149.log.gz 2026-02-22T23:21:00.708 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.66210.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48811.log 2026-02-22T23:21:00.708 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80484.log 2026-02-22T23:21:00.708 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.48811.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74215.log.gz 2026-02-22T23:21:00.708 INFO:teuthology.orchestra.run.trial082.stderr: 65.8% -- replaced with /var/log/ceph/cluster1-client.admin.48811.log.gz 2026-02-22T23:21:00.708 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.66210.log.gz 2026-02-22T23:21:00.708 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62035.log 2026-02-22T23:21:00.708 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80484.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63886.log 2026-02-22T23:21:00.709 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.80484.log.gz 2026-02-22T23:21:00.709 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.62035.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85291.log 2026-02-22T23:21:00.709 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.62035.log.gz 2026-02-22T23:21:00.709 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.63886.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62683.log 2026-02-22T23:21:00.709 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63886.log.gz 2026-02-22T23:21:00.709 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.85291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64496.log 2026-02-22T23:21:00.709 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85291.log.gz 2026-02-22T23:21:00.709 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.62683.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.84192.log 2026-02-22T23:21:00.710 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.64496.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62463.log 2026-02-22T23:21:00.710 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62683.log.gz 2026-02-22T23:21:00.710 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.2.84192.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.64496.log.gz 2026-02-22T23:21:00.710 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58071.log 2026-02-22T23:21:00.710 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62463.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83839.log 2026-02-22T23:21:00.711 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.58071.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.62463.log.gz 2026-02-22T23:21:00.711 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.49602.log 2026-02-22T23:21:00.711 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83839.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72078.log 2026-02-22T23:21:00.711 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.58071.log.gz 2026-02-22T23:21:00.711 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.83839.log.gz 2026-02-22T23:21:00.711 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.49602.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69229.log 2026-02-22T23:21:00.711 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72078.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83774.log 2026-02-22T23:21:00.711 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72078.log.gz 2026-02-22T23:21:00.712 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.69229.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61000.log 2026-02-22T23:21:00.712 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83774.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64077.log 2026-02-22T23:21:00.712 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69229.log.gz 2026-02-22T23:21:00.712 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61000.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48197.log 2026-02-22T23:21:00.712 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83774.log.gz 2026-02-22T23:21:00.712 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.61000.log.gz 2026-02-22T23:21:00.712 INFO:teuthology.orchestra.run.trial082.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.3.49602.log.gz 2026-02-22T23:21:00.712 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56173.log 2026-02-22T23:21:00.713 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.64077.log: /var/log/ceph/cluster1-client.admin.48197.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48197.log.gz 2026-02-22T23:21:00.713 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57890.log 87.5% 2026-02-22T23:21:00.713 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.64077.log.gz 2026-02-22T23:21:00.713 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.56173.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68769.log 2026-02-22T23:21:00.713 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.57890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73868.log 2026-02-22T23:21:00.713 INFO:teuthology.orchestra.run.trial082.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.2.84192.log.gz 2026-02-22T23:21:00.713 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.68769.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54574.log 2026-02-22T23:21:00.714 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.68769.log.gz 2026-02-22T23:21:00.714 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73868.log: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56173.log.gz 2026-02-22T23:21:00.714 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.57890.log.gz 2026-02-22T23:21:00.714 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66157.log 2026-02-22T23:21:00.714 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.54574.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65833.log 2026-02-22T23:21:00.714 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73868.log.gz 2026-02-22T23:21:00.714 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.54574.log.gz 2026-02-22T23:21:00.714 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.66157.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66270.log 2026-02-22T23:21:00.714 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.66157.log.gz 2026-02-22T23:21:00.714 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.65833.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62789.log 2026-02-22T23:21:00.715 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.66270.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50081.log 2026-02-22T23:21:00.715 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65833.log.gz 2026-02-22T23:21:00.715 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.66270.log.gz 2026-02-22T23:21:00.715 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.62789.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.79344.log 2026-02-22T23:21:00.715 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.62789.log.gz 2026-02-22T23:21:00.715 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.50081.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67425.log 2026-02-22T23:21:00.715 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.50081.log.gz 2026-02-22T23:21:00.715 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.79344.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67157.log 2026-02-22T23:21:00.716 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.67425.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.79231.log 2026-02-22T23:21:00.716 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.67425.log.gz 2026-02-22T23:21:00.716 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.67157.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79798.log 2026-02-22T23:21:00.716 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.79231.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.79231.log.gz 2026-02-22T23:21:00.716 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51695.log 2026-02-22T23:21:00.716 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67157.log.gz 2026-02-22T23:21:00.717 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.79798.log: /var/log/ceph/cluster1-client.mirror.75683.log 2026-02-22T23:21:00.717 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.79344.log.gz 2026-02-22T23:21:00.717 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51695.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76338.log 2026-02-22T23:21:00.717 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75683.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.51695.log.gz 2026-02-22T23:21:00.717 INFO:teuthology.orchestra.run.trial082.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.79798.log.gz 2026-02-22T23:21:00.717 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73247.log 2026-02-22T23:21:00.717 INFO:teuthology.orchestra.run.trial082.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75683.log.gz 2026-02-22T23:21:00.717 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76338.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50811.log 2026-02-22T23:21:00.718 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76338.log.gz 2026-02-22T23:21:00.718 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73247.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73247.log.gz 2026-02-22T23:21:00.718 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82662.log 2026-02-22T23:21:00.718 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.50811.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66703.log 2026-02-22T23:21:00.718 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.82662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56441.log 2026-02-22T23:21:00.718 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50811.log.gz 2026-02-22T23:21:00.718 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.66703.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82174.log 2026-02-22T23:21:00.718 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.56441.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68823.log 2026-02-22T23:21:00.719 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56441.log.gz 2026-02-22T23:21:00.719 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66703.log.gz 2026-02-22T23:21:00.719 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82174.log: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82662.log.gz 2026-02-22T23:21:00.719 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75123.log 2026-02-22T23:21:00.719 INFO:teuthology.orchestra.run.trial082.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82174.log.gz 2026-02-22T23:21:00.719 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68823.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70296.log 2026-02-22T23:21:00.719 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.68823.log.gz 2026-02-22T23:21:00.719 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75123.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77713.log 2026-02-22T23:21:00.719 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.70296.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61681.log 2026-02-22T23:21:00.720 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.75123.log.gz 2026-02-22T23:21:00.720 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.70296.log.gz 2026-02-22T23:21:00.720 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.77713.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75062.log 2026-02-22T23:21:00.720 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.61681.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.77713.log.gz 2026-02-22T23:21:00.720 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70323.log 2026-02-22T23:21:00.720 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61681.log.gz 2026-02-22T23:21:00.720 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75062.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76648.log 2026-02-22T23:21:00.720 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.70323.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.75062.log.gz 2026-02-22T23:21:00.721 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81883.log 2026-02-22T23:21:00.721 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.70323.log.gz 2026-02-22T23:21:00.721 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.76648.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80034.log 2026-02-22T23:21:00.721 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76648.log.gz 2026-02-22T23:21:00.721 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81883.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57288.log 2026-02-22T23:21:00.721 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80034.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81883.log.gz 2026-02-22T23:21:00.721 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66432.log 2026-02-22T23:21:00.722 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.57288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84033.log 2026-02-22T23:21:00.722 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.66432.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.57288.log.gz 2026-02-22T23:21:00.722 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82468.log 2026-02-22T23:21:00.722 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66432.log.gz 2026-02-22T23:21:00.722 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59905.log 2026-02-22T23:21:00.722 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.80034.log.gz 2026-02-22T23:21:00.722 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84033.log.gz 2026-02-22T23:21:00.722 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82468.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68308.log 2026-02-22T23:21:00.722 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.82468.log.gz 2026-02-22T23:21:00.723 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.59905.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78328.log 2026-02-22T23:21:00.723 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.59905.log.gz 2026-02-22T23:21:00.723 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.68308.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59092.log 2026-02-22T23:21:00.723 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.78328.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55932.log 2026-02-22T23:21:00.723 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.78328.log.gz 2026-02-22T23:21:00.723 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.59092.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.68308.log.gz 2026-02-22T23:21:00.723 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69738.log 2026-02-22T23:21:00.724 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.59092.log.gz 2026-02-22T23:21:00.724 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.55932.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87723.log 2026-02-22T23:21:00.724 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.69738.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77376.log 2026-02-22T23:21:00.724 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55932.log.gz 2026-02-22T23:21:00.724 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.69738.log.gz 2026-02-22T23:21:00.724 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.87723.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51055.log 2026-02-22T23:21:00.724 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.77376.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64239.log 2026-02-22T23:21:00.724 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.77376.log.gz 2026-02-22T23:21:00.725 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.51055.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47707.log 2026-02-22T23:21:00.725 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.64239.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.44217.log 2026-02-22T23:21:00.725 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64239.log.gz 2026-02-22T23:21:00.725 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47707.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47707.log.gz 2026-02-22T23:21:00.725 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.84260.log 2026-02-22T23:21:00.725 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51055.log.gz 2026-02-22T23:21:00.725 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.44217.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.44217.log.gz 2026-02-22T23:21:00.725 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61303.log 2026-02-22T23:21:00.726 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82517.log 2026-02-22T23:21:00.726 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.3.84260.log: /var/log/ceph/cluster1-client.mirror.61303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82404.log 2026-02-22T23:21:00.726 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82517.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.61303.log.gz 2026-02-22T23:21:00.726 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- 87.0% /var/log/ceph/cluster1-client.admin.64442.log -- replaced with /var/log/ceph/cluster1-client.mirror.82517.log.gz 2026-02-22T23:21:00.726 INFO:teuthology.orchestra.run.trial082.stderr: 2026-02-22T23:21:00.726 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.82404.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85877.log 2026-02-22T23:21:00.727 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.64442.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67677.log 2026-02-22T23:21:00.727 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.64442.log.gz 2026-02-22T23:21:00.727 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.85877.log: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87723.log.gz 2026-02-22T23:21:00.727 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57063.log 2026-02-22T23:21:00.727 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.85877.log.gz 2026-02-22T23:21:00.727 INFO:teuthology.orchestra.run.trial082.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82404.log.gz 2026-02-22T23:21:00.727 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.67677.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53443.log 2026-02-22T23:21:00.727 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.67677.log.gz 2026-02-22T23:21:00.728 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.57063.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64920.log 2026-02-22T23:21:00.728 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.57063.log.gz 2026-02-22T23:21:00.728 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.53443.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53443.log.gz 2026-02-22T23:21:00.728 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78726.log 2026-02-22T23:21:00.728 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.64920.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82128.log 2026-02-22T23:21:00.728 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.78726.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61985.log 2026-02-22T23:21:00.728 INFO:teuthology.orchestra.run.trial082.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78726.log.gz 2026-02-22T23:21:00.729 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.82128.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64920.log.gz 2026-02-22T23:21:00.729 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60030.log 2026-02-22T23:21:00.729 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.61985.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74751.log 2026-02-22T23:21:00.729 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60030.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64317.log 2026-02-22T23:21:00.729 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.60030.log.gz 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61985.log.gz 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74751.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82128.log.gz 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.3.84260.log.gz 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73160.log 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74751.log.gz 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.64317.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67623.log 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.64317.log.gz 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73160.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82371.log 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73160.log.gz 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67623.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54367.log 2026-02-22T23:21:00.730 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.67623.log.gz 2026-02-22T23:21:00.731 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.82371.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69470.log 2026-02-22T23:21:00.731 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.82371.log.gz 2026-02-22T23:21:00.731 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.54367.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78306.log 2026-02-22T23:21:00.731 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.54367.log.gz 2026-02-22T23:21:00.731 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.69470.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71219.log 2026-02-22T23:21:00.731 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.78306.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72734.log 2026-02-22T23:21:00.731 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.78306.log.gz 2026-02-22T23:21:00.732 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69470.log.gz 2026-02-22T23:21:00.732 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71219.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75942.log 2026-02-22T23:21:00.732 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71219.log.gz 2026-02-22T23:21:00.732 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48828.log 2026-02-22T23:21:00.732 INFO:teuthology.orchestra.run.trial082.stderr: 90.2%/var/log/ceph/cluster2-client.mirror.75942.log: -- replaced with /var/log/ceph/cluster1-client.mirror.72734.log.gz 2026-02-22T23:21:00.732 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67207.log 2026-02-22T23:21:00.732 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.48828.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75942.log.gz 2026-02-22T23:21:00.732 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59597.log 53.5% 2026-02-22T23:21:00.732 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.48828.log.gz 2026-02-22T23:21:00.733 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.67207.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56923.log 2026-02-22T23:21:00.733 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67207.log.gz 2026-02-22T23:21:00.733 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.59597.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59722.log 2026-02-22T23:21:00.733 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.59597.log.gz 2026-02-22T23:21:00.733 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.56923.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.53993.log 2026-02-22T23:21:00.733 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.56923.log.gz 2026-02-22T23:21:00.734 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.59722.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.49476.log 2026-02-22T23:21:00.734 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.59722.log.gz 2026-02-22T23:21:00.734 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.3.53993.log: 19.0% -- replaced with /var/log/ceph/cluster2-client.mirror.3.53993.log.gz 2026-02-22T23:21:00.734 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54928.log 2026-02-22T23:21:00.734 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.49476.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62436.log 2026-02-22T23:21:00.734 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.49476.log.gz 2026-02-22T23:21:00.734 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.54928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54661.log 2026-02-22T23:21:00.734 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.54928.log.gz 2026-02-22T23:21:00.734 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62436.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85998.log 2026-02-22T23:21:00.735 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.54661.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.62436.log.gz 2026-02-22T23:21:00.735 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82085.log 2026-02-22T23:21:00.735 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.85998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86653.log 2026-02-22T23:21:00.735 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.85998.log.gz 2026-02-22T23:21:00.735 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62517.log 2026-02-22T23:21:00.735 INFO:teuthology.orchestra.run.trial082.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54661.log.gz 2026-02-22T23:21:00.736 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.86653.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83386.log 2026-02-22T23:21:00.736 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.62517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43767.log 2026-02-22T23:21:00.736 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.82085.log.gz 2026-02-22T23:21:00.736 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.83386.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.62517.log.gz 2026-02-22T23:21:00.736 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.86653.log.gz 2026-02-22T23:21:00.736 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61167.log 2026-02-22T23:21:00.736 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.83386.log.gz 2026-02-22T23:21:00.736 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.43767.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83227.log 2026-02-22T23:21:00.736 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43767.log.gz 2026-02-22T23:21:00.737 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61167.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86102.log 2026-02-22T23:21:00.737 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83227.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.61167.log.gz 2026-02-22T23:21:00.737 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83446.log 2026-02-22T23:21:00.737 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.83227.log.gz 2026-02-22T23:21:00.737 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.86102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86046.log 2026-02-22T23:21:00.737 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86102.log.gz 2026-02-22T23:21:00.737 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.83446.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71136.log 2026-02-22T23:21:00.738 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.86046.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59538.log 2026-02-22T23:21:00.738 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.71136.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86046.log.gz 2026-02-22T23:21:00.738 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75728.log 2026-02-22T23:21:00.738 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71136.log.gz 2026-02-22T23:21:00.738 INFO:teuthology.orchestra.run.trial082.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83446.log.gz 2026-02-22T23:21:00.738 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.59538.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76600.log 2026-02-22T23:21:00.739 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75728.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.59538.log.gz 2026-02-22T23:21:00.739 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76089.log 2026-02-22T23:21:00.739 INFO:teuthology.orchestra.run.trial082.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75728.log.gz 2026-02-22T23:21:00.739 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76600.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74123.log 2026-02-22T23:21:00.739 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76600.log.gz 2026-02-22T23:21:00.739 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76089.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71491.log 2026-02-22T23:21:00.739 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74123.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76089.log.gz 2026-02-22T23:21:00.739 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61656.log 2026-02-22T23:21:00.740 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.71491.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51569.log 2026-02-22T23:21:00.740 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.61656.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74522.log 2026-02-22T23:21:00.740 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74123.log.gz 2026-02-22T23:21:00.740 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61656.log.gz 2026-02-22T23:21:00.740 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51569.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71491.log.gz 2026-02-22T23:21:00.740 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86022.log 2026-02-22T23:21:00.740 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.51569.log.gz 2026-02-22T23:21:00.741 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74522.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69048.log 2026-02-22T23:21:00.741 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.86022.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54499.log 2026-02-22T23:21:00.741 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86022.log.gz 2026-02-22T23:21:00.741 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.69048.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74522.log.gz 2026-02-22T23:21:00.741 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71158.log 2026-02-22T23:21:00.741 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.54499.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.49421.log 2026-02-22T23:21:00.741 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54499.log.gz 2026-02-22T23:21:00.741 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71158.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75551.log 2026-02-22T23:21:00.742 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.69048.log.gz 2026-02-22T23:21:00.742 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.49421.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71158.log.gz 2026-02-22T23:21:00.742 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49874.log 2026-02-22T23:21:00.742 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.49421.log.gz 2026-02-22T23:21:00.742 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75551.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84710.log 2026-02-22T23:21:00.742 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75551.log.gz 2026-02-22T23:21:00.742 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49874.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54523.log 2026-02-22T23:21:00.743 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.84710.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58243.log 2026-02-22T23:21:00.743 INFO:teuthology.orchestra.run.trial082.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49874.log.gz 2026-02-22T23:21:00.743 INFO:teuthology.orchestra.run.trial082.stderr: 79.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84710.log.gz 2026-02-22T23:21:00.743 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.54523.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.87082.log 2026-02-22T23:21:00.743 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.58243.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54523.log.gz 2026-02-22T23:21:00.743 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48947.log 2026-02-22T23:21:00.743 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.87082.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.87082.log.gz 2026-02-22T23:21:00.743 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49191.log 2026-02-22T23:21:00.744 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58243.log.gz 2026-02-22T23:21:00.744 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.48947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61479.log 2026-02-22T23:21:00.744 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48947.log.gz 2026-02-22T23:21:00.744 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49191.log: gzip 3.5% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.49191.log.gz --verbose 2026-02-22T23:21:00.744 INFO:teuthology.orchestra.run.trial082.stderr: -- /var/log/ceph/cluster1-client.mirror.0.76955.log 2026-02-22T23:21:00.744 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87808.log 2026-02-22T23:21:00.744 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61479.log.gz 2026-02-22T23:21:00.744 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.76955.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73378.log 2026-02-22T23:21:00.745 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.87808.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75428.log 2026-02-22T23:21:00.745 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73378.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87808.log.gz 2026-02-22T23:21:00.745 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81460.log 2026-02-22T23:21:00.745 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73378.log.gz 2026-02-22T23:21:00.745 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75428.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73827.log 2026-02-22T23:21:00.745 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81460.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75428.log.gz 2026-02-22T23:21:00.746 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56264.log 2026-02-22T23:21:00.746 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73827.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87670.log 2026-02-22T23:21:00.746 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.56264.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74235.log 2026-02-22T23:21:00.746 INFO:teuthology.orchestra.run.trial082.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73827.log.gz 2026-02-22T23:21:00.746 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.81460.log.gz 2026-02-22T23:21:00.746 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.56264.log.gz 2026-02-22T23:21:00.746 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.87670.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.38569.log 2026-02-22T23:21:00.747 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.87670.log.gz 2026-02-22T23:21:00.747 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74235.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75751.log 2026-02-22T23:21:00.747 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/ceph.tmp-client.admin.38569.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.38569.log.gz 2026-02-22T23:21:00.747 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81509.log 2026-02-22T23:21:00.747 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75751.log: gzip 91.5% -- replaced with /var/log/ceph/cluster1-client.mirror.0.76955.log.gz -5 2026-02-22T23:21:00.747 INFO:teuthology.orchestra.run.trial082.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.73990.log 2026-02-22T23:21:00.747 INFO:teuthology.orchestra.run.trial082.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75751.log.gz 2026-02-22T23:21:00.747 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81509.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53678.log 2026-02-22T23:21:00.748 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73990.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48001.log 2026-02-22T23:21:00.748 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.74235.log.gz 2026-02-22T23:21:00.748 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73990.log.gz 2026-02-22T23:21:00.748 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53678.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58185.log 2026-02-22T23:21:00.748 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.81509.log.gz 2026-02-22T23:21:00.748 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.48001.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63560.log 2026-02-22T23:21:00.748 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48001.log.gz 2026-02-22T23:21:00.748 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58185.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86363.log 2026-02-22T23:21:00.748 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.58185.log.gz 2026-02-22T23:21:00.749 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63560.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51100.log 2026-02-22T23:21:00.749 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.63560.log.gz 2026-02-22T23:21:00.749 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.86363.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62238.log 2026-02-22T23:21:00.749 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.86363.log.gz 2026-02-22T23:21:00.749 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.51100.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83307.log 2026-02-22T23:21:00.749 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62238.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80968.log 2026-02-22T23:21:00.749 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.62238.log.gz 2026-02-22T23:21:00.750 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83307.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51075.log 2026-02-22T23:21:00.750 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.83307.log.gz 2026-02-22T23:21:00.750 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.80968.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62388.log 2026-02-22T23:21:00.750 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.51075.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68382.log 2026-02-22T23:21:00.750 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80968.log.gz 2026-02-22T23:21:00.750 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.62388.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62113.log 2026-02-22T23:21:00.751 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.62388.log.gz 2026-02-22T23:21:00.751 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53678.log.gz 2026-02-22T23:21:00.751 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68382.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84979.log 2026-02-22T23:21:00.751 INFO:teuthology.orchestra.run.trial082.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.51075.log.gz 2026-02-22T23:21:00.751 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62113.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.68382.log.gz 2026-02-22T23:21:00.751 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70869.log 2026-02-22T23:21:00.751 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.62113.log.gz 2026-02-22T23:21:00.751 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.84979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55309.log 2026-02-22T23:21:00.751 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.84979.log.gz 2026-02-22T23:21:00.751 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.70869.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76750.log 2026-02-22T23:21:00.752 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.70869.log.gz 2026-02-22T23:21:00.752 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.55309.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87615.log 2026-02-22T23:21:00.752 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.76750.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51100.log.gz 2026-02-22T23:21:00.752 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59298.log 2026-02-22T23:21:00.752 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.55309.log.gz 2026-02-22T23:21:00.752 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.76750.log.gz 2026-02-22T23:21:00.752 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.87615.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65753.log 2026-02-22T23:21:00.752 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.87615.log.gz 2026-02-22T23:21:00.753 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.59298.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67807.log 2026-02-22T23:21:00.753 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.59298.log.gz 2026-02-22T23:21:00.753 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65753.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63426.log 2026-02-22T23:21:00.753 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.65753.log.gz 2026-02-22T23:21:00.753 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67807.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76068.log 2026-02-22T23:21:00.753 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.67807.log.gz 2026-02-22T23:21:00.753 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.63426.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.85598.log 2026-02-22T23:21:00.754 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76068.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65146.log 2026-02-22T23:21:00.754 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63426.log.gz 2026-02-22T23:21:00.754 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76068.log.gz 2026-02-22T23:21:00.754 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.85598.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71673.log 2026-02-22T23:21:00.754 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65146.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55153.log 2026-02-22T23:21:00.754 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.65146.log.gz 2026-02-22T23:21:00.754 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71882.log 2026-02-22T23:21:00.755 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.55153.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61846.log 2026-02-22T23:21:00.755 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.55153.log.gz 2026-02-22T23:21:00.755 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71882.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56570.log 2026-02-22T23:21:00.755 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71882.log.gz 2026-02-22T23:21:00.755 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61846.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84432.log 2026-02-22T23:21:00.755 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61846.log.gz 2026-02-22T23:21:00.755 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.56570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49979.log 2026-02-22T23:21:00.756 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.56570.log.gz 2026-02-22T23:21:00.756 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.84432.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84432.log.gz 2026-02-22T23:21:00.756 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65253.log 2026-02-22T23:21:00.756 INFO:teuthology.orchestra.run.trial082.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.85598.log.gz 2026-02-22T23:21:00.756 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79657.log 2026-02-22T23:21:00.756 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49979.log.gz 2026-02-22T23:21:00.756 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.65253.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57513.log 2026-02-22T23:21:00.756 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.79657.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56896.log 2026-02-22T23:21:00.757 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65253.log.gz 2026-02-22T23:21:00.757 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.57513.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.79657.log.gz 2026-02-22T23:21:00.757 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74296.log 2026-02-22T23:21:00.757 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.56896.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75348.log 2026-02-22T23:21:00.757 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.56896.log.gz 2026-02-22T23:21:00.757 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71673.log.gz 2026-02-22T23:21:00.757 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74296.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57513.log.gz 2026-02-22T23:21:00.757 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75163.log 2026-02-22T23:21:00.758 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74296.log.gz 2026-02-22T23:21:00.758 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75348.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72617.log 2026-02-22T23:21:00.758 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75163.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.80478.log 2026-02-22T23:21:00.758 INFO:teuthology.orchestra.run.trial082.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.50406.log.gz 2026-02-22T23:21:00.758 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75348.log.gz 2026-02-22T23:21:00.758 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75163.log.gz 2026-02-22T23:21:00.758 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72617.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52437.log 2026-02-22T23:21:00.758 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.80478.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88331.log 2026-02-22T23:21:00.758 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52437.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72617.log.gz 2026-02-22T23:21:00.759 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74337.log 2026-02-22T23:21:00.759 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.52437.log.gz 2026-02-22T23:21:00.759 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88331.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88331.log.gz 2026-02-22T23:21:00.759 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79700.log 2026-02-22T23:21:00.759 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74337.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65479.log 2026-02-22T23:21:00.759 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.79700.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74337.log.gz 2026-02-22T23:21:00.759 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78602.log 2026-02-22T23:21:00.759 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.79700.log.gz 2026-02-22T23:21:00.760 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.65479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72964.log 2026-02-22T23:21:00.760 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.78602.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74623.log 2026-02-22T23:21:00.760 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72964.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.3.80478.log.gz 2026-02-22T23:21:00.760 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65479.log.gz 2026-02-22T23:21:00.760 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74731.log 2026-02-22T23:21:00.760 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.72964.log.gz 2026-02-22T23:21:00.760 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74623.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54628.log 2026-02-22T23:21:00.761 INFO:teuthology.orchestra.run.trial082.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78602.log.gz 2026-02-22T23:21:00.761 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.74731.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81862.log 2026-02-22T23:21:00.761 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.54628.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.74731.log.gz 2026-02-22T23:21:00.761 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87040.log 2026-02-22T23:21:00.761 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.54628.log.gz 2026-02-22T23:21:00.761 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81862.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.85538.log 2026-02-22T23:21:00.761 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.87040.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63286.log 2026-02-22T23:21:00.762 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87040.log.gz 2026-02-22T23:21:00.762 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.85538.log: gzip -5 --verbose 19.6% -- -- replaced with /var/log/ceph/cluster1-client.mirror.3.85538.log.gz /var/log/ceph/cluster2-client.mirror.76047.log 2026-02-22T23:21:00.762 INFO:teuthology.orchestra.run.trial082.stderr: 2026-02-22T23:21:00.762 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.63286.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59146.log 2026-02-22T23:21:00.762 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.63286.log.gz 2026-02-22T23:21:00.762 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70475.log 2026-02-22T23:21:00.763 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76047.log: /var/log/ceph/cluster1-client.admin.59146.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.50451.log 2026-02-22T23:21:00.763 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76047.log.gz 2026-02-22T23:21:00.763 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.59146.log.gz 2026-02-22T23:21:00.763 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70475.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81981.log 2026-02-22T23:21:00.763 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.70475.log.gz 2026-02-22T23:21:00.763 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.50451.log: gzip -5 --verbose 19.6% -- /var/log/ceph/cluster1-client.admin.55852.log -- replaced with /var/log/ceph/cluster1-client.mirror.2.50451.log.gz 2026-02-22T23:21:00.763 INFO:teuthology.orchestra.run.trial082.stderr: 2026-02-22T23:21:00.763 INFO:teuthology.orchestra.run.trial082.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74623.log.gz 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81981.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77794.log 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.81981.log.gz 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.55852.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48979.log 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.55852.log.gz 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.77794.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78542.log 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81862.log.gz 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.48979.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48979.log.gzgzip 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72342.log 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.77794.log.gz 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.78542.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54681.log 2026-02-22T23:21:00.764 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.78542.log.gz 2026-02-22T23:21:00.765 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72342.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79772.log 2026-02-22T23:21:00.765 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.54681.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53084.log 2026-02-22T23:21:00.765 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72342.log.gz 2026-02-22T23:21:00.765 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.79772.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75388.log 2026-02-22T23:21:00.765 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.53084.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.79772.log.gz 2026-02-22T23:21:00.765 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65021.log 2026-02-22T23:21:00.766 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.53084.log.gz 2026-02-22T23:21:00.766 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75388.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80838.log 2026-02-22T23:21:00.766 INFO:teuthology.orchestra.run.trial082.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54681.log.gz 2026-02-22T23:21:00.766 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65021.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.84087.log 2026-02-22T23:21:00.766 INFO:teuthology.orchestra.run.trial082.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75388.log.gz 2026-02-22T23:21:00.766 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80838.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.65021.log.gz 2026-02-22T23:21:00.766 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76194.log 2026-02-22T23:21:00.766 INFO:teuthology.orchestra.run.trial082.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.80838.log.gz 2026-02-22T23:21:00.766 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69711.log 2026-02-22T23:21:00.766 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.0.84087.log: /var/log/ceph/cluster2-client.mirror.76194.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.79346.log 2026-02-22T23:21:00.767 INFO:teuthology.orchestra.run.trial082.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76194.log.gz 2026-02-22T23:21:00.767 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.69711.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88723.log 2026-02-22T23:21:00.767 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.79346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61054.log 2026-02-22T23:21:00.767 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.69711.log.gz 2026-02-22T23:21:00.767 INFO:teuthology.orchestra.run.trial082.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.79346.log.gz 2026-02-22T23:21:00.767 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88723.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74014.log 2026-02-22T23:21:00.767 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88723.log.gz 2026-02-22T23:21:00.767 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61054.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65540.log 2026-02-22T23:21:00.767 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.61054.log.gz 2026-02-22T23:21:00.768 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.74014.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84880.log 2026-02-22T23:21:00.768 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.74014.log.gz 2026-02-22T23:21:00.768 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65540.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77865.log 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.84880.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.65540.log.gz 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.85423.log 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84880.log.gz 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.77865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69765.log 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77865.log.gz 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.0.85423.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53797.log 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.85423.log.gz 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.69765.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67887.log 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.53797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71199.log 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.69765.log.gz 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53797.log.gz/var/log/ceph/cluster2-client.mirror.67887.log: 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81535.log 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71199.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70502.log 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81535.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67887.log.gz 2026-02-22T23:21:00.769 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59985.log 2026-02-22T23:21:00.770 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.70502.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63155.log 88.2% 2026-02-22T23:21:00.770 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.81535.log.gz 2026-02-22T23:21:00.770 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71199.log.gz 2026-02-22T23:21:00.770 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.70502.log.gz 2026-02-22T23:21:00.770 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.59985.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71922.log 2026-02-22T23:21:00.770 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.63155.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60949.log 2026-02-22T23:21:00.770 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59985.log.gz 2026-02-22T23:21:00.770 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.71922.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48862.log 2026-02-22T23:21:00.770 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63155.log.gz 2026-02-22T23:21:00.770 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.60949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55803.log 2026-02-22T23:21:00.771 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60949.log.gz 2026-02-22T23:21:00.771 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.48862.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52274.log 2026-02-22T23:21:00.771 INFO:teuthology.orchestra.run.trial082.stderr: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48862.log.gz 2026-02-22T23:21:00.771 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.55803.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64970.log 2026-02-22T23:21:00.771 INFO:teuthology.orchestra.run.trial082.stderr: 87.4%/var/log/ceph/cluster1-client.admin.52274.log: -- replaced with /var/log/ceph/cluster1-client.admin.55803.log.gz 2026-02-22T23:21:00.771 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52967.log 2026-02-22T23:21:00.771 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61279.log/var/log/ceph/cluster1-client.mirror.64970.log: 2026-02-22T23:21:00.771 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.52274.log.gz 2026-02-22T23:21:00.771 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52967.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.64970.log.gz 2026-02-22T23:21:00.771 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49745.log 2026-02-22T23:21:00.772 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.52967.log.gz 2026-02-22T23:21:00.772 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.61279.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56318.log 2026-02-22T23:21:00.772 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61279.log.gz 2026-02-22T23:21:00.772 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49745.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57916.log 2026-02-22T23:21:00.772 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.56318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74259.log 2026-02-22T23:21:00.772 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.57916.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.56318.log.gz 2026-02-22T23:21:00.772 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49745.log.gz 2026-02-22T23:21:00.772 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67548.log 2026-02-22T23:21:00.772 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57916.log.gz 2026-02-22T23:21:00.772 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60195.log 2026-02-22T23:21:00.773 INFO:teuthology.orchestra.run.trial082.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74259.log.gz 2026-02-22T23:21:00.773 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.67548.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61027.log 2026-02-22T23:21:00.773 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67548.log.gz 2026-02-22T23:21:00.773 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.60195.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68534.log 2026-02-22T23:21:00.773 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.60195.log.gz 2026-02-22T23:21:00.773 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61027.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43928.log 2026-02-22T23:21:00.773 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68534.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.61027.log.gz 2026-02-22T23:21:00.773 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68264.log 2026-02-22T23:21:00.773 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.43928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76110.log 2026-02-22T23:21:00.774 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.68534.log.gz 2026-02-22T23:21:00.774 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43928.log.gz 2026-02-22T23:21:00.774 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.68264.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72102.log 2026-02-22T23:21:00.774 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.76110.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66563.log 2026-02-22T23:21:00.774 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72102.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49242.log 2026-02-22T23:21:00.774 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76110.log.gz 2026-02-22T23:21:00.774 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.68264.log.gz 2026-02-22T23:21:00.774 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.66563.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.72102.log.gz 2026-02-22T23:21:00.774 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80988.log 2026-02-22T23:21:00.774 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.49242.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50318.log 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr: 57.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49242.log.gz 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.80988.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72776.log 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.66563.log.gz 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80988.log.gz 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.50318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46781.log 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50318.log.gz 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72776.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78665.log 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71922.log.gz 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.46781.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75183.log 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46781.log.gz 2026-02-22T23:21:00.775 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73626.log 2026-02-22T23:21:00.776 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75183.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51389.log 2026-02-22T23:21:00.776 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.73626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85949.log 2026-02-22T23:21:00.776 INFO:teuthology.orchestra.run.trial082.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75183.log.gz 2026-02-22T23:21:00.776 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72776.log.gz 2026-02-22T23:21:00.776 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51389.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83275.log 2026-02-22T23:21:00.776 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.85949.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72884.log 2026-02-22T23:21:00.776 INFO:teuthology.orchestra.run.trial082.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51389.log.gz 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.85949.log.gz 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83275.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60973.log 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.78665.log.gz 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83275.log.gz 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72884.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51722.log 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73626.log.gz 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60973.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75290.log 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72884.log.gz 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71336.log 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.60973.log.gz 2026-02-22T23:21:00.777 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.51722.log.gz 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50861.log 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.75290.log: /var/log/ceph/cluster2-client.mirror.71336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80817.log 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75290.log.gz 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71336.log.gz 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.50861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.87195.log 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50861.log.gz 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.80817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73765.log 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.87195.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67758.log 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.80817.log.gz 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73765.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83935.log 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73765.log.gz 2026-02-22T23:21:00.778 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67758.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81665.log 2026-02-22T23:21:00.779 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.67758.log.gz/var/log/ceph/cluster1-client.admin.83935.log: 2026-02-22T23:21:00.779 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55728.log 2026-02-22T23:21:00.779 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81665.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.83935.log.gz 2026-02-22T23:21:00.779 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62741.log 2026-02-22T23:21:00.779 INFO:teuthology.orchestra.run.trial082.stderr: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.81665.log.gz 2026-02-22T23:21:00.779 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.55728.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58725.log 2026-02-22T23:21:00.779 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.62741.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58317.log 2026-02-22T23:21:00.779 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62741.log.gz 2026-02-22T23:21:00.779 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58725.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.87195.log.gz 2026-02-22T23:21:00.779 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65372.log 2026-02-22T23:21:00.780 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.58725.log.gz 2026-02-22T23:21:00.780 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58317.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60703.log 2026-02-22T23:21:00.780 INFO:teuthology.orchestra.run.trial082.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55728.log.gz 2026-02-22T23:21:00.780 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65372.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63640.log 2026-02-22T23:21:00.780 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.58317.log.gz 2026-02-22T23:21:00.780 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.60703.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.65372.log.gzgzip 2026-02-22T23:21:00.780 INFO:teuthology.orchestra.run.trial082.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73270.log 2026-02-22T23:21:00.780 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.63640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46861.log 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr: 90.1%/var/log/ceph/cluster1-client.mirror.73270.log: -- replaced with /var/log/ceph/cluster2-client.mirror.60703.log.gz 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81909.log 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.73270.log.gz 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.46861.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63640.log.gz 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76277.log 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46861.log.gz 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61140.log 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81909.log.gz 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.76277.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53493.log 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.76277.log.gz 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59748.log 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.61140.log.gz 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.53493.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.85536.log 2026-02-22T23:21:00.781 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53493.log.gz 2026-02-22T23:21:00.782 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.59748.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54394.log 2026-02-22T23:21:00.782 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.2.85536.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65227.log 2026-02-22T23:21:00.782 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.54394.log: gzip -5 --verbose -- 89.7% /var/log/ceph/cluster2-client.admin.61327.log -- replaced with /var/log/ceph/cluster2-client.mirror.59748.log.gz 2026-02-22T23:21:00.782 INFO:teuthology.orchestra.run.trial082.stderr: 2026-02-22T23:21:00.782 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.54394.log.gz 2026-02-22T23:21:00.782 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.65227.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52001.log 2026-02-22T23:21:00.782 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.65227.log.gz 2026-02-22T23:21:00.783 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74172.log 2026-02-22T23:21:00.783 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61327.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52384.log 2026-02-22T23:21:00.783 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52001.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.61327.log.gz 2026-02-22T23:21:00.783 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74172.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51668.log 2026-02-22T23:21:00.783 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.52384.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.52001.log.gz 2026-02-22T23:21:00.783 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63862.log 2026-02-22T23:21:00.783 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.51668.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.79279.log 2026-02-22T23:21:00.783 INFO:teuthology.orchestra.run.trial082.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.52384.log.gz 2026-02-22T23:21:00.784 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.51668.log.gz 2026-02-22T23:21:00.784 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.63862.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74172.log.gz 2026-02-22T23:21:00.784 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48050.log 2026-02-22T23:21:00.784 INFO:teuthology.orchestra.run.trial082.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.85536.log.gz 2026-02-22T23:21:00.784 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63862.log.gz 2026-02-22T23:21:00.784 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70782.log 2026-02-22T23:21:00.784 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.79279.log: /var/log/ceph/cluster1-client.admin.48050.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48050.log.gz 2026-02-22T23:21:00.784 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69990.log 2026-02-22T23:21:00.785 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.70782.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68721.log 2026-02-22T23:21:00.785 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.69990.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55748.log 2026-02-22T23:21:00.785 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76723.log 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.68721.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.69990.log.gz 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.55748.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.70782.log.gz 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68721.log.gz 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56465.log 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.76723.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53348.log 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.56465.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73438.log 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.76723.log.gz 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56465.log.gz 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53348.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81008.log 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55748.log.gz 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53348.log.gz/var/log/ceph/cluster1-client.mirror.73438.log: 2026-02-22T23:21:00.786 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83533.log 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81008.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63836.log 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.73438.log.gz 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.81008.log.gz 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.83533.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78891.log 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83533.log.gz 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.79279.log.gz 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.63836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88674.log 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.78891.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70815.log 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78891.log.gz 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88674.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64946.log 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88674.log.gz 2026-02-22T23:21:00.787 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.63836.log.gz 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70815.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63319.log 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.64946.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.70815.log.gz 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60596.log 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64946.log.gz 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.63319.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60802.log 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.60596.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.63319.log.gz 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77515.log 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.60802.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72513.log 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.60596.log.gz 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.60802.log.gz 2026-02-22T23:21:00.788 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.77515.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60748.log 2026-02-22T23:21:00.789 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72513.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81731.log 2026-02-22T23:21:00.789 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.77515.log.gz 2026-02-22T23:21:00.789 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60748.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50941.log 2026-02-22T23:21:00.789 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81731.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.60748.log.gz 2026-02-22T23:21:00.789 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72003.log 2026-02-22T23:21:00.789 INFO:teuthology.orchestra.run.trial082.stderr: 79.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81731.log.gz 2026-02-22T23:21:00.789 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.50941.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62816.log 2026-02-22T23:21:00.789 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.50941.log.gz 2026-02-22T23:21:00.789 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58131.log 2026-02-22T23:21:00.789 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.62816.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63991.log 2026-02-22T23:21:00.790 INFO:teuthology.orchestra.run.trial082.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72513.log.gz 2026-02-22T23:21:00.790 INFO:teuthology.orchestra.run.trial082.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72003.log.gz 87.7% 2026-02-22T23:21:00.790 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.62816.log.gz/var/log/ceph/cluster1-client.admin.58131.log: 2026-02-22T23:21:00.790 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.82967.log 2026-02-22T23:21:00.790 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.58131.log.gz 2026-02-22T23:21:00.791 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65399.log 2026-02-22T23:21:00.791 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63991.log: /var/log/ceph/cluster1-client.mirror.3.82967.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58811.log 2026-02-22T23:21:00.791 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% 19.0% -- replaced with /var/log/ceph/cluster2-client.admin.63991.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.3.82967.log.gz 2026-02-22T23:21:00.791 INFO:teuthology.orchestra.run.trial082.stderr: 2026-02-22T23:21:00.791 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65399.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.81353.log 2026-02-22T23:21:00.791 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.65399.log.gz 2026-02-22T23:21:00.791 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58811.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78222.log 2026-02-22T23:21:00.792 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.81353.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.58811.log.gz 2026-02-22T23:21:00.792 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60057.log 2026-02-22T23:21:00.792 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.78222.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.81353.log.gz 2026-02-22T23:21:00.792 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59641.log 2026-02-22T23:21:00.792 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.60057.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.49535.log 2026-02-22T23:21:00.792 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.59641.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.60057.log.gz 2026-02-22T23:21:00.792 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77993.log 2026-02-22T23:21:00.792 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.59641.log.gz 2026-02-22T23:21:00.792 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.3.49535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68157.log 2026-02-22T23:21:00.792 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.49535.log.gz 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.77993.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.79242.log 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.68157.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63206.log 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.68157.log.gz 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77993.log.gz 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.1.79242.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.79242.log.gz 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67258.log 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69936.log 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.63206.log: /var/log/ceph/cluster2-client.admin.67258.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73075.log 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.63206.log.gz 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr: 87.5%/var/log/ceph/cluster1-client.admin.69936.log: -- replaced with /var/log/ceph/cluster2-client.admin.67258.log.gz 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75858.log 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.69936.log.gz/var/log/ceph/cluster2-client.mirror.73075.log: 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74917.log 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75858.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64888.log 2026-02-22T23:21:00.793 INFO:teuthology.orchestra.run.trial082.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73075.log.gz 2026-02-22T23:21:00.794 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74917.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75858.log.gz 2026-02-22T23:21:00.794 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76483.log 2026-02-22T23:21:00.794 INFO:teuthology.orchestra.run.trial082.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78222.log.gz 2026-02-22T23:21:00.794 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69098.log 2026-02-22T23:21:00.794 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.64888.log: /var/log/ceph/cluster2-client.mirror.76483.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50355.log 2026-02-22T23:21:00.794 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.69098.log: 87.5%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.76483.log.gz -5 2026-02-22T23:21:00.794 INFO:teuthology.orchestra.run.trial082.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.78382.log 2026-02-22T23:21:00.794 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69098.log.gz 2026-02-22T23:21:00.794 INFO:teuthology.orchestra.run.trial082.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64888.log.gz 2026-02-22T23:21:00.794 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.50355.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80659.log 2026-02-22T23:21:00.795 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50355.log.gz 2026-02-22T23:21:00.795 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.78382.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71438.log 2026-02-22T23:21:00.795 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.78382.log.gz 2026-02-22T23:21:00.795 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80659.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83570.log 2026-02-22T23:21:00.795 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.71438.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.80659.log.gz 2026-02-22T23:21:00.795 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73137.log 2026-02-22T23:21:00.795 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.71438.log.gz 2026-02-22T23:21:00.795 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79835.log 2026-02-22T23:21:00.795 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.83570.log: /var/log/ceph/cluster1-client.mirror.73137.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80884.log 2026-02-22T23:21:00.796 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83570.log.gz 2026-02-22T23:21:00.796 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.73137.log.gz 2026-02-22T23:21:00.796 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.79835.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57641.log 2026-02-22T23:21:00.796 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80884.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.79835.log.gz 2026-02-22T23:21:00.796 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78966.log 2026-02-22T23:21:00.796 INFO:teuthology.orchestra.run.trial082.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.80884.log.gz 2026-02-22T23:21:00.796 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60084.log 2026-02-22T23:21:00.796 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57641.log: /var/log/ceph/cluster1-client.admin.78966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86207.log 2026-02-22T23:21:00.796 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.57641.log.gz 2026-02-22T23:21:00.797 INFO:teuthology.orchestra.run.trial082.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74917.log.gz 2026-02-22T23:21:00.797 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.60084.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62870.log 2026-02-22T23:21:00.797 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.78966.log.gz 2026-02-22T23:21:00.797 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.60084.log.gz 2026-02-22T23:21:00.797 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.86207.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47609.log 2026-02-22T23:21:00.797 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.86207.log.gz 2026-02-22T23:21:00.797 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62870.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69835.log 2026-02-22T23:21:00.797 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47609.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.62870.log.gz 2026-02-22T23:21:00.797 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.53933.log 2026-02-22T23:21:00.797 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47609.log.gz 2026-02-22T23:21:00.798 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.69835.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57160.log 2026-02-22T23:21:00.798 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.2.53933.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58936.log 2026-02-22T23:21:00.798 INFO:teuthology.orchestra.run.trial082.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.2.53933.log.gz 2026-02-22T23:21:00.798 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.57160.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81025.log 2026-02-22T23:21:00.798 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.58936.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77464.log 2026-02-22T23:21:00.798 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.69835.log.gz 2026-02-22T23:21:00.798 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81025.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.58936.log.gz 2026-02-22T23:21:00.798 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46931.log 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.57160.log.gz 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.77464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57539.log 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81025.log.gz 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.77464.log.gz 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.46931.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72658.log 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46931.log.gz 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.57539.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.87897.log 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57539.log.gz 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.72658.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57991.log 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.87897.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48491.log 2026-02-22T23:21:00.799 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.87897.log.gz 2026-02-22T23:21:00.800 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72658.log.gz 2026-02-22T23:21:00.800 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57991.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71028.log 2026-02-22T23:21:00.800 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.48491.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.57991.log.gz 2026-02-22T23:21:00.800 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47658.log 2026-02-22T23:21:00.800 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48491.log.gz 2026-02-22T23:21:00.800 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80749.log 2026-02-22T23:21:00.800 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.71028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74788.log 2026-02-22T23:21:00.800 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.47658.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47658.log.gz 2026-02-22T23:21:00.800 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.71028.log.gz 2026-02-22T23:21:00.800 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80749.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53110.log 2026-02-22T23:21:00.801 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.80749.log.gz 2026-02-22T23:21:00.801 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61506.log 2026-02-22T23:21:00.801 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.53110.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66184.log 87.8% 2026-02-22T23:21:00.801 INFO:teuthology.orchestra.run.trial082.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.74788.log.gz 2026-02-22T23:21:00.801 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.61506.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76777.log 2026-02-22T23:21:00.801 INFO:teuthology.orchestra.run.trial082.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53110.log.gz 2026-02-22T23:21:00.801 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.66184.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55539.log 2026-02-22T23:21:00.801 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61506.log.gz 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.76777.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.66184.log.gz 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55381.log 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.55539.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83914.log 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.76777.log.gz 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.55381.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57234.log 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.83914.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.55381.log.gz 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69684.log 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55539.log.gz 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.57234.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.83914.log.gz 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75489.log 2026-02-22T23:21:00.802 INFO:teuthology.orchestra.run.trial082.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.57234.log.gz 2026-02-22T23:21:00.803 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63233.log 2026-02-22T23:21:00.803 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.69684.log: /var/log/ceph/cluster2-client.mirror.75489.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62011.log 2026-02-22T23:21:00.803 INFO:teuthology.orchestra.run.trial082.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.69684.log.gz 2026-02-22T23:21:00.803 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.63233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81228.log 2026-02-22T23:21:00.803 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.62011.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.63233.log.gz 2026-02-22T23:21:00.803 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69520.log 2026-02-22T23:21:00.803 INFO:teuthology.orchestra.run.trial082.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62011.log.gz 2026-02-22T23:21:00.803 INFO:teuthology.orchestra.run.trial082.stderr: 95.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.84087.log.gz 2026-02-22T23:21:00.803 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69360.log 2026-02-22T23:21:00.804 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.81228.log: /var/log/ceph/cluster1-client.mirror.69520.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63048.log 2026-02-22T23:21:00.804 INFO:teuthology.orchestra.run.trial082.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75489.log.gz 2026-02-22T23:21:00.804 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.69520.log.gz 2026-02-22T23:21:00.804 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.69360.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64157.log 2026-02-22T23:21:00.804 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81228.log.gz 2026-02-22T23:21:00.804 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.63048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59119.log 2026-02-22T23:21:00.804 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.69360.log.gz 2026-02-22T23:21:00.804 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.64157.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72798.log 2026-02-22T23:21:00.804 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.63048.log.gz 2026-02-22T23:21:00.804 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.59119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55985.log 2026-02-22T23:21:00.805 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.72798.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.59119.log.gz 2026-02-22T23:21:00.805 INFO:teuthology.orchestra.run.trial082.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64157.log.gz 2026-02-22T23:21:00.805 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65878.log 2026-02-22T23:21:00.805 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.55985.log: gzip 90.7% -5 -- replaced with /var/log/ceph/cluster1-client.mirror.72798.log.gz --verbose 2026-02-22T23:21:00.805 INFO:teuthology.orchestra.run.trial082.stderr: -- /var/log/ceph/cluster1-client.admin.70377.log 2026-02-22T23:21:00.805 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.65878.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48148.log 2026-02-22T23:21:00.805 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.65878.log.gz 2026-02-22T23:21:00.806 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.70377.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55985.log.gz 2026-02-22T23:21:00.806 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77821.log 2026-02-22T23:21:00.806 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.70377.log.gz 2026-02-22T23:21:00.806 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.48148.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73542.log 2026-02-22T23:21:00.806 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48148.log.gz 2026-02-22T23:21:00.806 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.77821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50335.log 2026-02-22T23:21:00.806 INFO:teuthology.orchestra.run.trial082.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.77821.log.gz 2026-02-22T23:21:00.806 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.73542.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72417.log 2026-02-22T23:21:00.807 INFO:teuthology.orchestra.run.trial082.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.73542.log.gz 2026-02-22T23:21:00.807 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.50335.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48794.log 2026-02-22T23:21:00.807 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.72417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53057.log 2026-02-22T23:21:00.807 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50335.log.gz 2026-02-22T23:21:00.807 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.48794.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61533.log 2026-02-22T23:21:00.807 INFO:teuthology.orchestra.run.trial082.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.admin.48794.log.gz 2026-02-22T23:21:00.807 INFO:teuthology.orchestra.run.trial082.stderr: 90.7% -- replaced with /var/log/ceph/cluster1-client.admin.72417.log.gz 2026-02-22T23:21:00.808 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.53057.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58457.log 2026-02-22T23:21:00.808 INFO:teuthology.orchestra.run.trial082.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.53057.log.gz 2026-02-22T23:21:00.808 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61533.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54790.log 2026-02-22T23:21:00.808 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.61533.log.gz 2026-02-22T23:21:00.808 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.58457.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78746.log 2026-02-22T23:21:00.808 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.58457.log.gz 2026-02-22T23:21:00.809 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.54790.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61354.log 2026-02-22T23:21:00.809 INFO:teuthology.orchestra.run.trial082.stderr: 86.9% -- replaced with /var/log/ceph/cluster2-client.admin.54790.log.gz 2026-02-22T23:21:00.809 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.78746.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81181.log 2026-02-22T23:21:00.809 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.61354.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57186.log 2026-02-22T23:21:00.809 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.61354.log.gz 2026-02-22T23:21:00.809 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.81181.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67312.log 2026-02-22T23:21:00.809 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.81181.log.gz 2026-02-22T23:21:00.810 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.57186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71055.log 2026-02-22T23:21:00.810 INFO:teuthology.orchestra.run.trial082.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.57186.log.gz 2026-02-22T23:21:00.810 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.67312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82241.log 2026-02-22T23:21:00.810 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.71055.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.67312.log.gz 2026-02-22T23:21:00.810 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74276.log 2026-02-22T23:21:00.810 INFO:teuthology.orchestra.run.trial082.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.71055.log.gz 2026-02-22T23:21:00.810 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.82241.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-22T23:21:00.811 INFO:teuthology.orchestra.run.trial082.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82241.log.gz 2026-02-22T23:21:00.811 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74276.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74869.log 2026-02-22T23:21:00.811 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66863.log 2026-02-22T23:21:00.811 INFO:teuthology.orchestra.run.trial082.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74276.log.gz 2026-02-22T23:21:00.811 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.74869.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79168.log 2026-02-22T23:21:00.812 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.66863.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.74869.log.gz 2026-02-22T23:21:00.812 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78515.log 2026-02-22T23:21:00.812 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.79168.log: 87.4%gzip -- replaced with /var/log/ceph/cluster2-client.admin.66863.log.gz -5 2026-02-22T23:21:00.812 INFO:teuthology.orchestra.run.trial082.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.80281.log 2026-02-22T23:21:00.812 INFO:teuthology.orchestra.run.trial082.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78746.log.gz 2026-02-22T23:21:00.812 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.78515.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88429.log 2026-02-22T23:21:00.812 INFO:teuthology.orchestra.run.trial082.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79168.log.gz 2026-02-22T23:21:00.812 INFO:teuthology.orchestra.run.trial082.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.78515.log.gz 2026-02-22T23:21:00.812 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.80281.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62211.log 2026-02-22T23:21:00.813 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.80281.log.gz 2026-02-22T23:21:00.813 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.admin.88429.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86533.log 2026-02-22T23:21:00.813 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.88429.log.gz 2026-02-22T23:21:00.813 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.62211.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56949.log 2026-02-22T23:21:00.813 INFO:teuthology.orchestra.run.trial082.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.62211.log.gz 2026-02-22T23:21:00.813 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69598.log 2026-02-22T23:21:00.813 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.86533.log: /var/log/ceph/cluster2-client.mirror.56949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53773.log 2026-02-22T23:21:00.814 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.admin.69598.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.56949.log.gz 2026-02-22T23:21:00.814 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75510.log 2026-02-22T23:21:00.814 INFO:teuthology.orchestra.run.trial082.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.86533.log.gz 2026-02-22T23:21:00.814 INFO:teuthology.orchestra.run.trial082.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.69598.log.gz 2026-02-22T23:21:00.814 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster1-client.mirror.53773.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53773.log.gz 2026-02-22T23:21:00.815 INFO:teuthology.orchestra.run.trial082.stderr:/var/log/ceph/cluster2-client.mirror.75510.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75510.log.gz 2026-02-22T23:21:01.011 INFO:teuthology.orchestra.run.trial060.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-22T23:21:01.177 INFO:teuthology.orchestra.run.trial082.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-22T23:21:05.074 INFO:teuthology.orchestra.run.trial060.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-22T23:21:05.193 INFO:teuthology.orchestra.run.trial060.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-22T23:21:05.324 INFO:teuthology.orchestra.run.trial082.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-22T23:21:05.725 INFO:teuthology.orchestra.run.trial082.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-22T23:21:05.788 INFO:teuthology.orchestra.run.trial060.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-22T23:21:05.790 INFO:teuthology.orchestra.run.trial060.stderr: 2026-02-22T23:21:05.790 INFO:teuthology.orchestra.run.trial060.stderr:real 0m5.373s 2026-02-22T23:21:05.790 INFO:teuthology.orchestra.run.trial060.stderr:user 0m14.864s 2026-02-22T23:21:05.790 INFO:teuthology.orchestra.run.trial060.stderr:sys 0m0.690s 2026-02-22T23:21:06.163 INFO:teuthology.orchestra.run.trial082.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-22T23:21:06.165 INFO:teuthology.orchestra.run.trial082.stderr: 2026-02-22T23:21:06.165 INFO:teuthology.orchestra.run.trial082.stderr:real 0m5.726s 2026-02-22T23:21:06.165 INFO:teuthology.orchestra.run.trial082.stderr:user 0m16.338s 2026-02-22T23:21:06.165 INFO:teuthology.orchestra.run.trial082.stderr:sys 0m1.505s 2026-02-22T23:21:06.166 INFO:tasks.ceph:Archiving logs... 2026-02-22T23:21:06.166 DEBUG:teuthology.misc:Transferring archived files from trial060:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284/remote/trial060/log 2026-02-22T23:21:06.167 DEBUG:teuthology.orchestra.run.trial060:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T23:21:07.354 DEBUG:teuthology.misc:Transferring archived files from trial082:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284/remote/trial082/log 2026-02-22T23:21:07.355 DEBUG:teuthology.orchestra.run.trial082:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T23:21:09.346 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-22T23:21:09.397 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-22T23:21:09.397 DEBUG:teuthology.orchestra.run.trial060:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T23:21:09.401 DEBUG:teuthology.orchestra.run.trial082:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T23:21:09.586 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-22T23:21:09.604 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-22T23:21:09.605 DEBUG:teuthology.orchestra.run.trial060:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T23:21:09.607 DEBUG:teuthology.orchestra.run.trial082:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T23:21:09.621 INFO:teuthology.orchestra.run.trial060.stderr:bash: line 1: ntpq: command not found 2026-02-22T23:21:09.623 INFO:teuthology.orchestra.run.trial082.stderr:bash: line 1: ntpq: command not found 2026-02-22T23:21:09.655 INFO:teuthology.orchestra.run.trial060.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T23:21:09.655 INFO:teuthology.orchestra.run.trial060.stdout:=============================================================================== 2026-02-22T23:21:09.655 INFO:teuthology.orchestra.run.trial060.stdout:^- 172-234-44-141.ip.linode> 2 7 377 112 +15ms[ +13ms] +/- 94ms 2026-02-22T23:21:09.655 INFO:teuthology.orchestra.run.trial060.stdout:^* time-ord.far.net 2 7 377 47 -703us[-3464us] +/- 25ms 2026-02-22T23:21:09.655 INFO:teuthology.orchestra.run.trial060.stdout:^- ntp01-baq.provider0.inf 2 7 377 49 +2646us[ -116us] +/- 117ms 2026-02-22T23:21:09.655 INFO:teuthology.orchestra.run.trial060.stdout:^+ ntp1.glypnod.com 2 7 377 237 +1772us[ -950us] +/- 47ms 2026-02-22T23:21:09.658 INFO:teuthology.orchestra.run.trial082.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T23:21:09.658 INFO:teuthology.orchestra.run.trial082.stdout:=============================================================================== 2026-02-22T23:21:09.658 INFO:teuthology.orchestra.run.trial082.stdout:^- ntp2.zeus.frumentum.media 2 6 377 42 -106ms[ -106ms] +/- 129ms 2026-02-22T23:21:09.658 INFO:teuthology.orchestra.run.trial082.stdout:^- 72-14-186-59.ip.linodeus> 2 7 377 109 -2847us[-2818us] +/- 92ms 2026-02-22T23:21:09.658 INFO:teuthology.orchestra.run.trial082.stdout:^- 173.249.203.227 2 7 377 110 +1739us[+1768us] +/- 32ms 2026-02-22T23:21:09.658 INFO:teuthology.orchestra.run.trial082.stdout:^* nyc3.us.ntp.li 2 6 377 42 +70us[ +102us] +/- 4240us 2026-02-22T23:21:09.658 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-22T23:21:09.669 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-22T23:21:09.669 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-22T23:21:09.682 DEBUG:teuthology.orchestra.run.trial060:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-22T23:21:09.749 DEBUG:teuthology.orchestra.run.trial082:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-22T23:21:09.805 DEBUG:teuthology.orchestra.run.trial060:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T23:21:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T23:21:09.833 DEBUG:teuthology.orchestra.run.trial082:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T23:21:09.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T23:21:09.860 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-22T23:21:09.880 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-22T23:21:09.898 INFO:teuthology.task.internal:Duration was 1770.358612 seconds 2026-02-22T23:21:09.898 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-22T23:21:09.910 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-22T23:21:09.910 DEBUG:teuthology.orchestra.run.trial060:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T23:21:09.912 DEBUG:teuthology.orchestra.run.trial082:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T23:21:09.946 INFO:teuthology.orchestra.run.trial060.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T23:21:09.947 INFO:teuthology.orchestra.run.trial082.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T23:21:10.152 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-22T23:21:10.153 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial060.front.sepia.ceph.com 2026-02-22T23:21:10.153 DEBUG:teuthology.orchestra.run.trial060:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T23:21:10.173 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial082.front.sepia.ceph.com 2026-02-22T23:21:10.173 DEBUG:teuthology.orchestra.run.trial082:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T23:21:10.210 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-22T23:21:10.210 DEBUG:teuthology.orchestra.run.trial060:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T23:21:10.214 DEBUG:teuthology.orchestra.run.trial082:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T23:21:10.314 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-22T23:21:10.314 DEBUG:teuthology.orchestra.run.trial060:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T23:21:10.317 DEBUG:teuthology.orchestra.run.trial082:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T23:21:10.339 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T23:21:10.339 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T23:21:10.339 INFO:teuthology.orchestra.run.trial060.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T23:21:10.340 INFO:teuthology.orchestra.run.trial060.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T23:21:10.340 INFO:teuthology.orchestra.run.trial060.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T23:21:10.341 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T23:21:10.341 INFO:teuthology.orchestra.run.trial082.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T23:21:10.342 INFO:teuthology.orchestra.run.trial082.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T23:21:10.342 INFO:teuthology.orchestra.run.trial082.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T23:21:10.342 INFO:teuthology.orchestra.run.trial082.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T23:21:10.350 INFO:teuthology.orchestra.run.trial060.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.3% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T23:21:10.352 INFO:teuthology.orchestra.run.trial082.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.4% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T23:21:10.354 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-22T23:21:10.371 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-22T23:21:10.372 DEBUG:teuthology.orchestra.run.trial060:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T23:21:10.415 DEBUG:teuthology.orchestra.run.trial082:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T23:21:10.439 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-22T23:21:10.456 DEBUG:teuthology.orchestra.run.trial060:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T23:21:10.459 DEBUG:teuthology.orchestra.run.trial082:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T23:21:10.481 INFO:teuthology.orchestra.run.trial060.stdout:kernel.core_pattern = core 2026-02-22T23:21:10.503 INFO:teuthology.orchestra.run.trial082.stdout:kernel.core_pattern = core 2026-02-22T23:21:10.534 DEBUG:teuthology.orchestra.run.trial060:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T23:21:10.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T23:21:10.551 DEBUG:teuthology.orchestra.run.trial082:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T23:21:10.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T23:21:10.590 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-22T23:21:10.608 INFO:teuthology.task.internal:Transferring archived files... 2026-02-22T23:21:10.609 DEBUG:teuthology.misc:Transferring archived files from trial060:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284/remote/trial060 2026-02-22T23:21:10.609 DEBUG:teuthology.orchestra.run.trial060:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T23:21:10.647 DEBUG:teuthology.misc:Transferring archived files from trial082:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64284/remote/trial082 2026-02-22T23:21:10.647 DEBUG:teuthology.orchestra.run.trial082:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T23:21:10.682 INFO:teuthology.task.internal:Removing archive directory... 2026-02-22T23:21:10.683 DEBUG:teuthology.orchestra.run.trial060:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T23:21:10.684 DEBUG:teuthology.orchestra.run.trial082:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T23:21:10.731 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-22T23:21:10.749 INFO:teuthology.task.internal:Not uploading archives. 2026-02-22T23:21:10.749 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-22T23:21:10.767 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-22T23:21:10.767 DEBUG:teuthology.orchestra.run.trial060:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T23:21:10.770 DEBUG:teuthology.orchestra.run.trial082:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T23:21:10.785 INFO:teuthology.orchestra.run.trial060.stdout: 917572 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 23:21 /home/ubuntu/cephtest 2026-02-22T23:21:10.787 INFO:teuthology.orchestra.run.trial082.stdout: 917572 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 23:21 /home/ubuntu/cephtest 2026-02-22T23:21:10.788 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-22T23:21:10.804 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-22T23:21:10.846 INFO:teuthology.run:Summary data: description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-bitmap supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-exclusive-lock} duration: 1770.3586118221283 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-22T23:21:10.846 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T23:21:10.958 INFO:teuthology.run:pass